esse-async_indexing 0.0.2 → 0.1.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a7931a646008cb93875d4e0fc47a552c2a19e3f9089c5e500e3cbb822f32f9bd
-  data.tar.gz: 374d59d804ca1b37e30f853b765d5b6ea0b975a97335d45249fa6e9f019d171c
+  metadata.gz: 50b00ff520d7e52425dd2b97ff0ce4cf7f3f7c2bd8275ef2a36d290e9ca3165e
+  data.tar.gz: 5ad4d7445414dcc7c9cc670fa348ee26d4cd589aa71e9c84a331ce7d848e43b1
 SHA512:
-  metadata.gz: fb5ef232b141e0b2ee7f629f139376f4ef169da0b7078fdf6b5e74937219c096f065e5d0b982004ab0fe50b0b5363fd09dad62a1fab110f59d84d0e7f7072561
-  data.tar.gz: 7b09c24955e14df91435ef2a4ce39796c36ddb7ee0e9c173a40fbad0e7d11cf86c67b891454555f5814ebaf56994042a361de7c2fbd9ab093a7709a946aed0c2
+  metadata.gz: 20b7b5d815774e36d237d2273442546fa5289d45b4db187962c48da4e2f153219d1cc2439cb2461c75cdaff5b6ff0ceb8e27eacfe3c754f908a22300431c80f9
+  data.tar.gz: 538588ff73ce223fbfbfb46e19d7b47ee8da1131734f0e6228ff0142adf13324ade2f31b94209538e94e8b8fffa85b28ef9644dbc71f8dcb1874410046da8659

data/CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.1.0 - 2024-08-22
+* Remove esse-redis_storage dependency
+* Remove the batch id jobs and actions related redis storage
+* Add the --eager-load-lazy-attributes option to the async_import cli command
+* Add the --update-lazy-attributes option to the async_import cli command
+* Add the --enqueue-lazy-attributes option to the async_import cli command
+* Add the --preload-lazy-attributes option to the async_import cli command
+* Add the --job-options option to the async_import cli command
+* Update import related jobs to process ids instead of batch id
+
 ## 0.0.2 - 2024-08-02
 * Include sidekiq and faktory jobs to perform async indexing of documents
 * Create Active Model async indexing callbacks

data/Gemfile

@@ -2,11 +2,12 @@
 
 source "https://rubygems.org"
 
-gem "esse"
+gem "esse", github: "marcosgz/esse", branch: "main"
 gem "esse-rspec"
 gem "esse-active_record", ">= 0.3.5", require: false
 gem "timecop"
 gem "faktory_worker_ruby", require: false
 gem "opensearch-ruby", require: false
+gem "background_job"
 
 gemspec

data/Gemfile.lock

@@ -1,9 +1,18 @@
+GIT
+  remote: https://github.com/marcosgz/esse.git
+  revision: 8fd302274e9c9c0dcdeec7b8f6ffafc05234014b
+  branch: main
+  specs:
+    esse (0.4.0.rc1)
+      multi_json
+      thor (>= 0.19)
+
 PATH
   remote: .
   specs:
-    esse-async_indexing (0.0.2)
-      esse (>= 0.3.4)
-      esse-redis_storage (>= 0.1.0)
+    esse-async_indexing (0.1.0.rc1)
+      background_job
+      esse (>= 0.4.0.rc01)
       multi_json
 
 GEM
@@ -28,6 +37,9 @@ GEM
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
     ast (2.4.2)
+    background_job (0.0.1.rc3)
+      multi_json
+      redis
     base64 (0.2.0)
     bigdecimal (3.1.8)
     coderay (1.1.3)
@@ -38,16 +50,9 @@ GEM
       rexml
     diff-lcs (1.5.1)
     drb (2.2.1)
-    esse (0.3.4)
-      multi_json
-      thor (>= 0.19)
     esse-active_record (0.3.5)
       activerecord (>= 4.2, < 8)
       esse (>= 0.3.0)
-    esse-redis_storage (0.1.0)
-      esse (>= 0.3.2)
-      multi_json (>= 1.0.0)
-      redis (>= 4.0.0)
     esse-rspec (0.0.6)
       esse (>= 0.2.4)
       rspec (>= 3)
@@ -81,7 +86,7 @@ GEM
     public_suffix (5.1.1)
     racc (1.8.0)
     rainbow (3.1.1)
-    redis (5.2.0)
+    redis (5.3.0)
       redis-client (>= 0.22.0)
     redis-client (0.22.2)
       connection_pool
@@ -149,7 +154,8 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
-  esse
+  background_job
+  esse!
   esse-active_record (>= 0.3.5)
   esse-async_indexing!
   esse-rspec

data/README.md

@@ -21,17 +21,13 @@ $ bundle install
 
 ```ruby
 Esse.configure do |config|
-  config.redis = ConnectionPool.new(size: 10, timeout: 5) do
-    Redis.new(url: ENV.fetch('REDIS_URL', 'redis://0.0.0.0:6379'))
-  end
-
   # Setup Sidekiq
   require 'sidekiq'
   config.async_indexing.sidekiq do |sidekiq|
-    sidekiq.namespace = "sidekiq"
     sidekiq.redis = ConnectionPool.new(size: 10, timeout: 5) do
       Redis.new(url: ENV.fetch('REDIS_URL', 'redis://0.0.0.0:6379'))
     end
+    # sidekiq.namespace = "sidekiq" # Sidekiq recommends using redis db number instead of namespace, but you can use it if you want
   end
 
   # Faktory
@@ -40,34 +36,58 @@ Esse.configure do |config|
 end
 ```
 
-Optional worker configuration:
+### Configuration > Jobs Queues
+
+Set the queues for each job and other options like retry, timeout, etc:
 
 ```ruby
 Esse.configure do |config|
-  config.async_indexing.sidekiq.workers = {
+  config.async_indexing.sidekiq.jobs = {
+    "Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob" => { queue: "batch_indexing" },
     "Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob" => { queue: "indexing" },
     "Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob" => { queue: "indexing" },
     "Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob" => { queue: "indexing" },
     "Esse::AsyncIndexing::Jobs::DocumentUpsertByIdJob" => { queue: "indexing" },
-    "Esse::AsyncIndexing::Jobs::ImportAllJob" => { queue: "batch_indexing", retry: 3 },
-    "Esse::AsyncIndexing::Jobs::ImportBatchIdJob" => { queue: "batch_indexing", retry: 3 },
-    "Esse::AsyncIndexing::Jobs::BulkUpdateLazyDocumentAttributeJob" => { queue: "batch_indexing", retry: 3 },
-    "Esse::AsyncIndexing::Jobs::UpdateLazyDocumentAttributeJob" => { queue: "indexing" },
+    "Esse::AsyncIndexing::Jobs::ImportAllJob" => { queue: "batch_indexing", retry: false },
+    "Esse::AsyncIndexing::Jobs::ImportIdsJob" => { queue: "batch_indexing", retry: 2 },
   }
   # or if you are using Faktory
-  config.async_indexing.faktory.workers = {
+  config.async_indexing.faktory.jobs = {
+    "Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob" => { queue: "batch_indexing" },
     "Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob" => { queue: "indexing" },
     "Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob" => { queue: "indexing" },
     "Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob" => { queue: "indexing" },
     "Esse::AsyncIndexing::Jobs::DocumentUpsertByIdJob" => { queue: "indexing" },
-    "Esse::AsyncIndexing::Jobs::ImportAllJob" => { queue: "batch_indexing", retry: 3 },
-    "Esse::AsyncIndexing::Jobs::ImportBatchIdJob" => { queue: "batch_indexing", retry: 3 },
-    "Esse::AsyncIndexing::Jobs::BulkUpdateLazyDocumentAttributeJob" => { queue: "batch_indexing", retry: 3 },
-    "Esse::AsyncIndexing::Jobs::UpdateLazyDocumentAttributeJob" => { queue: "indexing" },
+    "Esse::AsyncIndexing::Jobs::ImportAllJob" => { queue: "batch_indexing", retry: false },
+    "Esse::AsyncIndexing::Jobs::ImportIdsJob" => { queue: "batch_indexing", retry: 2 },
   }
 end
 ```
 
+### Configuration > Tasks
+
+To overwrite the default job that is enqueued for each operation. The default jobs are:
+* :import => Esse::AsyncIndexing::Jobs::ImportIdsJob
+* :index => Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob
+* :update => Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob
+* :delete => Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob
+* :update_lazy_attribute => Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob
+
+The operation can be set globally using the `task` method or per index using the `async_indexing_job` method.
+
+Example above will first store ids in different storage and just enqueue job with the batch_id:
+
+```ruby
+Esse.configure do |config|
+  config.async_indexing.task(:import) do |service:, repo:, operation:, ids:, **kwargs|
+    batch_id = Esse::RedisStorage::Queue.for(repo: repo).enqueue(values: ids)
+    ImportBatchIdJob.perform_later(repo.index.name, repo.repo_name, batch_id, **kwargs)
+  end
+end
+```
+
+Now when calling the async_import CLI command, it will push jobs to the `ImportBatchIdJob` instead of the standard `Esse::AsyncIndexing::Jobs::ImportIdsJob`.
+
 ## Index Configuration
 
 To enable async indexing for an index, you need to add the `:async_indexing` plugin to the index. And the index collection must implement the `#each_batch_ids` method that yields an array of document ids.
@@ -107,13 +127,13 @@ $ bundle exec esse index async_import GeosIndex --suffix="20240101" --service="s
 
 ## Workers/Jobs
 
-The gem provides a few jobs to index, update, upsert and delete document or batch of documents with given ids. The sidekiq or faktory worker does not need to live in the same application that enqueues the job. The worker can be in a separate application that only runs the worker process. This gem has its own DSL to push jobs.
+The gem provides a few jobs to index, update, upsert and delete document or batch of documents with given ids. The sidekiq or faktory job does not need to live in the same application that enqueues the job. The job can be in a separate application that only runs the job process. This gem has its own DSL to push jobs.
 
-But for make sure to require the jobs in the worker application by calling `install!`
+But for make sure to require the jobs in the job application by calling `install!`
 
 ```ruby
-Esse::AsyncIndexing::Workers.install!(:faktory)
-Esse::AsyncIndexing::Workers.install!(:sidekiq)
+Esse::AsyncIndexing::Jobs.install!(:faktory)
+Esse::AsyncIndexing::Jobs.install!(:sidekiq)
 ```
 
 
@@ -122,7 +142,7 @@ Esse::AsyncIndexing::Workers.install!(:sidekiq)
 Fetch a document from `GeosIndex.repo(:city)` collection using the given id and index it
 
 ```ruby
-Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob", service: :sidekiq).with_args("GeosIndex", "city", city.id, suffix: "20240101")
+BackgroundJob.sidekiq("Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob").with_args("GeosIndex", "city", city.id, "suffix" => "20240101")
 .push
 ```
 
@@ -133,7 +153,7 @@ Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob", se
 Fetch a document from `GeosIndex.repo(:city)` collection using the given id and update it
 
 ```ruby
-Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob", service: :sidekiq).with_args("GeosIndex", "city", city.id, suffix: "20240101")
+BackgroundJob.sidekiq("Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob").with_args("GeosIndex", "city", city.id, "suffix" => "20240101")
 ```
 
 **Note:** Suffix is optional, just an example of how to pass additional arguments to the job.
@@ -143,7 +163,7 @@ Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob", s
 Fetch a document from `GeosIndex.repo(:city)` collection using the given id and upsert it
 
 ```ruby
-Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentUpsertByIdJob", service: :sidekiq).with_args("GeosIndex", "city", city.id, suffix: "20240101")
+BackgroundJob.sidekiq("Esse::AsyncIndexing::Jobs::DocumentUpsertByIdJob").with_args("GeosIndex", "city", city.id, "suffix" => "20240101")
 ```
 
 **Note:** Suffix is optional, just an example of how to pass additional arguments to the job.
@@ -154,7 +174,7 @@ Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentUpsertByIdJob", s
 Delete a document from the index using the given id
 
 ```ruby
-Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob", service: :sidekiq).with_args("GeosIndex", "city", city.id, suffix: "20240101")
+BackgroundJob.sidekiq("Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob").with_args("GeosIndex", "city", city.id, "suffix" => "20240101")
 ```
 
 **Note:** Suffix is optional, just an example of how to pass additional arguments to the job.
@@ -164,38 +184,27 @@ Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob", s
 Import all documents from the `GeosIndex.repo(:city)` collection where `state_abbr` is "IL"
 
 ```ruby
-Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::ImportAllJob", service: :sidekiq).with_args("GeosIndex", "city", context: {state_abbr: "IL"}, suffix: "20240101")
+BackgroundJob.sidekiq("Esse::AsyncIndexing::Jobs::ImportAllJob").with_args("GeosIndex", "city", "context" => { "state_abbr" => "IL"}, "suffix" => "20240101")
 ```
 
 **Note:** Suffix and import context are optional, just an example of how to pass additional arguments to the job.
 
-### Esse::AsyncIndexing::Jobs::ImportBatchIdJob
-
-Import a batch of documents from the `GeosIndex.repo(:city)` collection using a batch_id generated by the [esse-redis_storage](https://github.com/marcosgz/esse-redis_storage) gem. This is the job that the `async_import` command uses.
-
-```ruby
-batch_id = Esse::RedisStorage::Queue.for(repo: GeosIndex.repo(:city)).enqueue(values: big_list_of_uuids)
-Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::ImportBatchIdJob", service: :sidekiq).with_args("GeosIndex", "city", batch_id, suffix: "20240101")
-```
-**Note:** Suffix is optional, just an example of how to pass additional arguments to the job.
-
-### Esse::AsyncIndexing::Jobs::BulkUpdateLazyDocumentAttributeJob
+### Esse::AsyncIndexing::Jobs::ImportIdsJob
 
-Update a lazy attribute of a document from the index using the given enqueued batch_id.
+Import a batch of documents from the `GeosIndex.repo(:city)` collection using the given ids
 
 ```ruby
-batch_id = Esse::RedisStorage::Queue.for(repo: GeosIndex.repo(:city), attribute_name: "total_schools").enqueue(values: big_list_of_uuids)
-Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::BulkUpdateLazyDocumentAttributeJob", service: :sidekiq).with_args("GeosIndex", "city", "total_schools", batch_id, suffix: "20240101")
+BackgroundJob.sidekiq("Esse::AsyncIndexing::Jobs::ImportIdsJob").with_args("GeosIndex", "city", city_ids, "suffix" => "20240101")
 ```
 
 **Note:** Suffix is optional, just an example of how to pass additional arguments to the job.
 
-### Esse::AsyncIndexing::Jobs::UpdateLazyDocumentAttributeJob
+### Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob
 
 Update a lazy attribute of a document from the index using the given id
 
 ```ruby
-Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::UpdateLazyDocumentAttributeJob", service: :sidekiq).with_args("GeosIndex", "city", "total_schools", [city.id], suffix: "20240101")
+BackgroundJob.sidekiq("Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob").with_args("GeosIndex", "city", "total_schools", [city.id], "suffix" => "20240101")
 ```
 
 **Note:** Suffix is optional, just an example of how to pass additional arguments to the job.

data/lib/esse/async_indexing/actions/{batch_delete.rb → bulk_delete.rb}

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
 module Esse::AsyncIndexing::Actions
-  class BatchDelete
+  class BulkDelete
     def self.call(index_class_name, repo_name, ids, options = {})
       docs = Esse::LazyDocumentHeader.coerce_each(ids)
       return if docs.empty?
 
       index_class, _repo_class = CoerceIndexRepository.call(index_class_name, repo_name)
       bulk_opts = Esse::HashUtils.deep_transform_keys(options, &:to_sym)
-      index_class.bulk(**bulk_opts, delete: docs.map(&:to_doc))
+      index_class.bulk(**bulk_opts, delete: docs.map(&:doc_header))
       docs.size
     end
   end

data/lib/esse/async_indexing/actions/{batch_import.rb → bulk_import.rb}

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Esse::AsyncIndexing::Actions
-  class BatchImport
+  class BulkImport
     def self.call(index_class_name, repo_name, ids, options = {})
       ids = Array(ids)
       return if ids.empty?

data/lib/esse/async_indexing/actions/{batch_import_all.rb → bulk_import_all.rb}

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Esse::AsyncIndexing::Actions
-  class BatchImportAll
+  class BulkImportAll
     def self.call(index_class_name, repo_name, options = {})
       _index_class, repo_class = CoerceIndexRepository.call(index_class_name, repo_name)
       kwargs = Esse::HashUtils.deep_transform_keys(options, &:to_sym)

data/lib/esse/async_indexing/actions/{batch_update.rb → bulk_update.rb}

@@ -1,8 +1,7 @@
 # frozen_string_literal: true
 
 module Esse::AsyncIndexing::Actions
-  class BatchUpdate
-    DOC_ARGS = %i[lazy_attributes context]
+  class BulkUpdate
 
     def self.call(index_class_name, repo_name, ids, options = {})
       ids = Array(ids)
@@ -14,8 +13,11 @@ module Esse::AsyncIndexing::Actions
       if (context = bulk_opts.delete(:context))
         find_opts.merge!(context)
       end
-      if (lazy_attributes = bulk_opts.delete(:lazy_attributes))
-        find_opts[:lazy_attributes] = lazy_attributes
+      if (val = bulk_opts.delete(:eager_load_lazy_attributes))
+        find_opts[:eager_load_lazy_attributes] = val
+      end
+      if (val = bulk_opts.delete(:preload_lazy_attributes))
+        find_opts[:preload_lazy_attributes] = val
       end
       find_opts[:id] = ids
 

data/lib/esse/async_indexing/actions/{update_lazy_document_attribute.rb → bulk_update_lazy_attribute.rb}

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Esse::AsyncIndexing::Actions
-  class UpdateLazyDocumentAttribute
+  class BulkUpdateLazyAttribute
     def self.call(index_class_name, repo_name, attr_name, ids, options = {})
       _index_class, repo_class = CoerceIndexRepository.call(index_class_name, repo_name)
       kwargs = Esse::HashUtils.deep_transform_keys(options, &:to_sym)

data/lib/esse/async_indexing/actions/delete_document.rb

@@ -2,7 +2,7 @@
 
 module Esse::AsyncIndexing::Actions
   class DeleteDocument
-    DOC_ARGS = %i[lazy_attributes]
+    DOC_ARGS = %i[eager_load_lazy_attributes preload_lazy_attributes]
 
     def self.call(index_class_name, repo_name, document_id, options = {})
       index_class, _repo_class = CoerceIndexRepository.call(index_class_name, repo_name)

data/lib/esse/async_indexing/actions/index_document.rb

@@ -2,7 +2,7 @@
 
 module Esse::AsyncIndexing::Actions
   class IndexDocument
-    DOC_ARGS = %i[lazy_attributes]
+    DOC_ARGS = %i[eager_load_lazy_attributes preload_lazy_attributes]
 
     def self.call(index_class_name, repo_name, document_id, options = {})
       index_class, repo_class = CoerceIndexRepository.call(index_class_name, repo_name)

data/lib/esse/async_indexing/actions/update_document.rb

@@ -2,7 +2,7 @@
 
 module Esse::AsyncIndexing::Actions
   class UpdateDocument
-    DOC_ARGS = %i[lazy_attributes]
+    DOC_ARGS = %i[eager_load_lazy_attributes preload_lazy_attributes]
 
     def self.call(index_class_name, repo_name, document_id, options = {})
       index_class, repo_class = CoerceIndexRepository.call(index_class_name, repo_name)

data/lib/esse/async_indexing/actions/upsert_document.rb

@@ -2,8 +2,7 @@
 
 module Esse::AsyncIndexing::Actions
   class UpsertDocument
-    DOC_ARGS = %i[lazy_attributes]
-    # OPERATIONS = %w[index update delete]
+    DOC_ARGS = %i[eager_load_lazy_attributes preload_lazy_attributes]
 
     def self.call(index_class_name, repo_name, document_id, operation = "index", options = {})
       case operation

data/lib/esse/async_indexing/actions.rb

@@ -8,14 +8,12 @@ module Esse
 end
 
 require_relative "actions/coerce_index_repository"
-require_relative "actions/batch_import_all"
-require_relative "actions/batch_import"
-require_relative "actions/batch_delete"
-require_relative "actions/batch_update"
-require_relative "actions/bulk_update_lazy_document_attribute"
+require_relative "actions/bulk_delete"
+require_relative "actions/bulk_import_all"
+require_relative "actions/bulk_import"
+require_relative "actions/bulk_update_lazy_attribute"
+require_relative "actions/bulk_update"
 require_relative "actions/delete_document"
-require_relative "actions/import_batch_id"
 require_relative "actions/index_document"
 require_relative "actions/update_document"
 require_relative "actions/upsert_document"
-require_relative "actions/update_lazy_document_attribute"

data/lib/esse/async_indexing/active_record_callbacks/lazy_update_attribute.rb

@@ -3,8 +3,6 @@
 module Esse::AsyncIndexing
   module ActiveRecordCallbacks
     class LazyUpdateAttribute < Callback
-      LAZY_ATTR_WORKER = "Esse::AsyncIndexing::Jobs::UpdateLazyDocumentAttributeJob"
-
       attr_reader :attribute_name
 
       def initialize(service_name:, attribute_name:, with: nil, **kwargs)
@@ -14,9 +12,14 @@ module Esse::AsyncIndexing
 
       def call(model)
         if (doc_ids = resolve_document_ids(model))
-          Esse::AsyncIndexing.worker(LAZY_ATTR_WORKER, service: service_name)
-            .with_args(repo.index.name, repo.repo_name, attribute_name.to_s, doc_ids, Esse::HashUtils.deep_transform_keys(options, &:to_s))
-            .push
+          repo.async_indexing_job_for(:update_lazy_attribute).call(
+            service: service_name,
+            repo: repo,
+            operation: :update_lazy_attribute,
+            attribute: attribute_name,
+            ids: doc_ids,
+            **options
+          )
         end
 
         true

data/lib/esse/async_indexing/cli/async_import.rb

@@ -3,11 +3,25 @@
 require "esse/cli/index/base_operation"
 
 class Esse::AsyncIndexing::CLI::AsyncImport < Esse::CLI::Index::BaseOperation
-  WORKER_NAME = "Esse::AsyncIndexing::Jobs::ImportBatchIdJob"
+  WORKER_NAME = "Esse::AsyncIndexing::Jobs::ImportIdsJob"
+
+  attr_reader :job_options, :service_name
+
+  def initialize(indices:, job_options: {}, service: nil, **options)
+    @job_options = job_options
+    @service_name = (service || Esse.config.async_indexing.services.first)&.to_sym
+    super(indices: indices, **options)
+  end
 
   def run
     validate_options!
     indices.each do |index|
+      unless Esse::AsyncIndexing.plugin_installed?(index)
+        raise Esse::CLI::InvalidOption, <<~MSG
+          The #{index} index does not support async indexing. Make sure you have the `plugin :async_indexing` in your `#{index}` class.
+        MSG
+      end
+
       repos = if (repo = @options[:repo])
         [index.repo(repo)]
       else
@@ -17,23 +31,22 @@ class Esse::AsyncIndexing::CLI::AsyncImport < Esse::CLI::Index::BaseOperation
       repos.each do |repo|
         unless Esse::AsyncIndexing.async_indexing_repo?(repo)
           raise Esse::CLI::InvalidOption, <<~MSG
-            The #{repo} repository does not support async indexing. Make sure you have the `plugin :async_indexing` in your `#{index}` class and the :#{repo.repo_name} collection implements the `#each_batch_ids` method.
+            The #{repo} repository does not support async indexing. Make sure the :#{repo.repo_name} collection of `#{index}` implements the `#each_batch_ids` method.
           MSG
         end
 
-        enqueuer = if (caller = repo.async_indexing_jobs[:import])
-          ->(ids) { caller.call(service_name, repo, :import, ids, **bulk_options) }
-        else
-          queue = Esse::RedisStorage::Queue.for(repo: repo)
-          ->(ids) do
-            batch_id = queue.enqueue(values: ids)
-            Esse::AsyncIndexing.worker(WORKER_NAME, service: service_name)
-              .with_args(repo.index.name, repo.repo_name, batch_id, Esse::HashUtils.deep_transform_keys(bulk_options, &:to_s))
-              .push
+        repo.batch_ids(**bulk_options.fetch(:context, {})).each do |ids|
+          kwargs = {
+            service: service_name,
+            repo: repo,
+            operation: :import,
+            ids: ids,
+            **bulk_options
+          }.tap do |hash|
+            hash[:job_options] = job_options if job_options.any?
           end
+          repo.async_indexing_job_for(:import).call(**kwargs)
         end
-
-        repo.batch_ids.each(&enqueuer)
       end
     end
   end
@@ -42,7 +55,7 @@ class Esse::AsyncIndexing::CLI::AsyncImport < Esse::CLI::Index::BaseOperation
 
   def bulk_options
     @bulk_options ||= begin
-      hash = @options.slice(*@options.keys - Esse::CLI_IGNORE_OPTS - [:repo, :service])
+      hash = @options.slice(*@options.keys - Esse::CLI_IGNORE_OPTS - [:repo])
       hash.delete(:context) if hash[:context].nil? || hash[:context].empty?
       hash
     end
@@ -51,8 +64,4 @@ class Esse::AsyncIndexing::CLI::AsyncImport < Esse::CLI::Index::BaseOperation
   def validate_options!
     validate_indices_option!
   end
-
-  def service_name
-    (@options[:service] || Esse.config.async_indexing.services.first)&.to_sym
-  end
 end

data/lib/esse/async_indexing/cli/async_update_lazy_attributes.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "esse/cli/index/base_operation"
+
+class Esse::AsyncIndexing::CLI::AsyncUpdateLazyAttributes < Esse::CLI::Index::BaseOperation
+  WORKER_NAME = "Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob"
+
+  attr_reader :attributes, :job_options, :service_name
+
+  def initialize(indices:, attributes: nil, job_options: nil, service: nil, **options)
+    super(indices: indices, **options)
+    @attributes = Array(attributes)
+    @job_options = job_options || {}
+    @service_name = (service || Esse.config.async_indexing.services.first)&.to_sym
+  end
+
+  def run
+    validate_options!
+    indices.each do |index|
+      unless Esse::AsyncIndexing.plugin_installed?(index)
+        raise Esse::CLI::InvalidOption, <<~MSG
+          The #{index} index does not support async indexing. Make sure you have the `plugin :async_indexing` in your `#{index}` class.
+        MSG
+      end
+
+      repos = if (repo = @options[:repo])
+        [index.repo(repo)]
+      else
+        index.repo_hash.values
+      end
+
+      repos.each do |repo|
+        unless Esse::AsyncIndexing.async_indexing_repo?(repo)
+          raise Esse::CLI::InvalidOption, <<~MSG
+            The #{repo} repository does not support async indexing. Make sure the :#{repo.repo_name} collection of `#{index}` implements the `#each_batch_ids` method.
+          MSG
+        end
+
+        attrs = repo_attributes(repo)
+        next unless attrs.any?
+
+        repo.batch_ids(**@options.fetch(:context, {})).each do |ids|
+          kwargs = {
+            service: service_name,
+            repo: repo,
+            operation: :update_lazy_attribute,
+            ids: ids,
+            **bulk_options
+          }.tap do |hash|
+            hash[:job_options] = job_options if job_options.any?
+          end
+          attrs.each do |attribute|
+            repo.async_indexing_job_for(:update_lazy_attribute).call(**kwargs, attribute: attribute)
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  def bulk_options
+    @bulk_options ||= begin
+      hash = @options.slice(*@options.keys - Esse::CLI_IGNORE_OPTS - [:context, :repo])
+      hash
+    end
+  end
+
+  def validate_options!
+    validate_indices_option!
+  end
+
+  def repo_attributes(repo)
+    return repo.lazy_document_attributes.keys if attributes.empty?
+
+    repo.lazy_document_attribute_names(attributes)
+  end
+end