esse-async_indexing 0.0.2 → 0.1.0.rc1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +10 -0
  3. data/Gemfile +2 -1
  4. data/Gemfile.lock +18 -12
  5. data/README.md +50 -41
  6. data/lib/esse/async_indexing/actions/{batch_delete.rb → bulk_delete.rb} +2 -2
  7. data/lib/esse/async_indexing/actions/{batch_import.rb → bulk_import.rb} +1 -1
  8. data/lib/esse/async_indexing/actions/{batch_import_all.rb → bulk_import_all.rb} +1 -1
  9. data/lib/esse/async_indexing/actions/{batch_update.rb → bulk_update.rb} +6 -4
  10. data/lib/esse/async_indexing/actions/{update_lazy_document_attribute.rb → bulk_update_lazy_attribute.rb} +1 -1
  11. data/lib/esse/async_indexing/actions/delete_document.rb +1 -1
  12. data/lib/esse/async_indexing/actions/index_document.rb +1 -1
  13. data/lib/esse/async_indexing/actions/update_document.rb +1 -1
  14. data/lib/esse/async_indexing/actions/upsert_document.rb +1 -2
  15. data/lib/esse/async_indexing/actions.rb +5 -7
  16. data/lib/esse/async_indexing/active_record_callbacks/lazy_update_attribute.rb +8 -5
  17. data/lib/esse/async_indexing/cli/async_import.rb +27 -18
  18. data/lib/esse/async_indexing/cli/async_update_lazy_attributes.rb +78 -0
  19. data/lib/esse/async_indexing/cli.rb +23 -9
  20. data/lib/esse/async_indexing/config.rb +0 -3
  21. data/lib/esse/async_indexing/configuration.rb +54 -18
  22. data/lib/esse/async_indexing/jobs/bulk_update_lazy_attribute_job.rb +7 -0
  23. data/lib/esse/async_indexing/jobs/import_all_job.rb +1 -1
  24. data/lib/esse/async_indexing/jobs/import_ids_job.rb +42 -0
  25. data/lib/esse/async_indexing/{workers.rb → jobs.rb} +7 -24
  26. data/lib/esse/async_indexing/tasks.rb +82 -0
  27. data/lib/esse/async_indexing/version.rb +1 -1
  28. data/lib/esse/async_indexing.rb +15 -20
  29. data/lib/esse/plugins/async_indexing.rb +18 -59
  30. metadata +19 -32
  31. data/lib/esse/async_indexing/actions/bulk_update_lazy_document_attribute.rb +0 -20
  32. data/lib/esse/async_indexing/actions/import_batch_id.rb +0 -21
  33. data/lib/esse/async_indexing/adapters/adapter.rb +0 -29
  34. data/lib/esse/async_indexing/adapters/faktory.rb +0 -114
  35. data/lib/esse/async_indexing/adapters/sidekiq.rb +0 -94
  36. data/lib/esse/async_indexing/adapters.rb +0 -12
  37. data/lib/esse/async_indexing/configuration/base.rb +0 -65
  38. data/lib/esse/async_indexing/configuration/faktory.rb +0 -6
  39. data/lib/esse/async_indexing/configuration/sidekiq.rb +0 -12
  40. data/lib/esse/async_indexing/jobs/bulk_update_lazy_document_attribute_job.rb +0 -7
  41. data/lib/esse/async_indexing/jobs/import_batch_id_job.rb +0 -34
  42. data/lib/esse/async_indexing/jobs/update_lazy_document_attribute_job.rb +0 -7
  43. data/lib/esse/async_indexing/testing.rb +0 -79
  44. data/lib/esse/async_indexing/worker.rb +0 -85
  45. data/lib/esse/async_indexing/workers/faktory.rb +0 -28
  46. data/lib/esse/async_indexing/workers/shared_class_methods.rb +0 -26
  47. data/lib/esse/async_indexing/workers/sidekiq.rb +0 -28
@@ -13,20 +13,34 @@ Esse::CLI::Index.class_eval do
13
13
  option :suffix, type: :string, default: nil, aliases: "-s", desc: "Suffix to append to index name"
14
14
  option :context, type: :hash, default: {}, required: true, desc: "List of options to pass to the index class"
15
15
  option :service, type: :string, default: nil, alias: "-s", desc: "Service to use for async import: sidekiq, faktory"
16
- option :eager_include_document_attributes, type: :string, default: nil, desc: "Comma separated list of lazy document attributes to include to the bulk index request. Or pass `true` to include all lazy attributes"
17
- option :lazy_update_document_attributes, type: :string, default: nil, desc: "Comma separated list of lazy document attributes to bulk update after the bulk index request Or pass `true` to include all lazy attributes"
16
+ option :preload_lazy_attributes, type: :string, default: nil, desc: "Command separated list of lazy document attributes to preload using search API before the bulk import. Or pass `true` to preload all lazy attributes"
17
+ option :eager_load_lazy_attributes, type: :string, default: nil, desc: "Comma separated list of lazy document attributes to include to the bulk index request. Or pass `true` to include all lazy attributes"
18
+ option :update_lazy_attributes, type: :string, default: nil, desc: "Comma separated list of lazy document attributes to bulk update after the bulk index request Or pass `true` to include all lazy attributes"
19
+ option :enqueue_lazy_attributes, type: :boolean, default: nil, desc: "Enqueue the lazy document attributes job after the bulk import. (default: true))"
20
+ option :job_options, type: :hash, default: {}, desc: "List of options to pass to the background job. (Example: --job-options=queue:default)"
18
21
  def async_import(*index_classes)
19
22
  opts = Esse::HashUtils.deep_transform_keys(options.to_h, &:to_sym)
20
23
  opts[:service] ||= Esse.config.async_indexing.services.first
21
- opts.delete(:lazy_update_document_attributes) if opts[:lazy_update_document_attributes] == "false"
22
- opts.delete(:eager_include_document_attributes) if opts[:eager_include_document_attributes] == "false"
23
- if (val = opts[:eager_include_document_attributes])
24
- opts[:eager_include_document_attributes] = (val == "true") ? true : val.split(",")
25
- end
26
- if (val = opts[:lazy_update_document_attributes])
27
- opts[:lazy_update_document_attributes] = (val == "true") ? true : val.split(",")
24
+ %i[preload_lazy_attributes eager_load_lazy_attributes update_lazy_attributes].each do |key|
25
+ if (val = opts.delete(key)) && val != "false"
26
+ opts[key] = (val == "true") ? true : val.split(",")
27
+ end
28
28
  end
29
29
  require "esse/async_indexing/cli/async_import"
30
30
  Esse::AsyncIndexing::CLI::AsyncImport.new(indices: index_classes, **opts).run
31
31
  end
32
+
33
+ desc "async_update_lazy_attributes INDEX_CLASS", "Async update lazy attributes for the given index"
34
+ option :repo, type: :string, default: nil, alias: "-r", desc: "Repository to use for import"
35
+ option :suffix, type: :string, default: nil, aliases: "-s", desc: "Suffix to append to index name"
36
+ option :context, type: :hash, default: {}, required: true, desc: "List of options to pass to the index class"
37
+ option :service, type: :string, default: nil, alias: "-s", desc: "Service to use for async import: sidekiq, faktory"
38
+ option :job_options, type: :hash, default: {}, desc: "List of options to pass to the background job. (Example: --job-options=queue:default)"
39
+ def async_update_lazy_attributes(index_class, *attributes)
40
+ opts = Esse::HashUtils.deep_transform_keys(options.to_h, &:to_sym)
41
+ opts[:service] ||= Esse.config.async_indexing.services.first
42
+ require "esse/async_indexing/cli/async_update_lazy_attributes"
43
+
44
+ Esse::AsyncIndexing::CLI::AsyncUpdateLazyAttributes.new(indices: [index_class], attributes: attributes, **opts).run
45
+ end
32
46
  end
@@ -1,9 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "configuration"
4
- require_relative "configuration/base"
5
- require_relative "configuration/faktory"
6
- require_relative "configuration/sidekiq"
7
4
 
8
5
  module Esse
9
6
  module AsyncIndexing
@@ -17,29 +17,65 @@ module Esse
17
17
  @services ||= ConfigService.new
18
18
  end
19
19
 
20
- def faktory
21
- @faktory ||= begin
22
- services.add(:faktory)
23
- Configuration::Faktory.new
24
- end
25
- if block_given?
26
- yield @faktory
27
- else
28
- @faktory
29
- end
20
+ def reset!
21
+ @services = nil
22
+ @tasks = nil
23
+ bg_job_config.reset!
30
24
  end
31
25
 
32
- def sidekiq
33
- @sidekiq ||= begin
34
- services.add(:sidekiq)
35
- Configuration::Sidekiq.new
26
+ BackgroundJob::SERVICES.each_key do |service|
27
+ define_method(service) do |&block|
28
+ config_for(service, &block)
36
29
  end
37
- if block_given?
38
- yield @sidekiq
39
- else
40
- @sidekiq
30
+ end
31
+
32
+ def config_for(service)
33
+ case service.to_sym
34
+ when :faktory
35
+ conf = bg_job_config.faktory
36
+ unless services.faktory?
37
+ default_jobs.each { |job_class| conf.jobs[job_class] ||= {} }
38
+ services << :faktory
39
+ end
40
+ yield conf if block_given?
41
+ conf
42
+ when :sidekiq
43
+ conf = bg_job_config.sidekiq
44
+ unless services.sidekiq?
45
+ default_jobs.each { |job_class| conf.jobs[job_class] ||= {} }
46
+ services << :sidekiq
47
+ end
48
+ yield conf if block_given?
49
+ conf
50
+ else raise ArgumentError, "Unknown service: #{service}"
41
51
  end
42
52
  end
53
+
54
+ def tasks
55
+ @tasks ||= Esse::AsyncIndexing::Tasks.new
56
+ end
57
+
58
+ # DSL to define custom job enqueueing
59
+ #
60
+ # task(:import) do |service:, repo:, operation:, ids:, **kwargs|
61
+ # MyCustomJob.perform_later(repo.index.name, ids, **kwargs)
62
+ # end
63
+ # task(:index, :update, :delete) do |service:, repo:, operation:, id, **kwargs|
64
+ # MyCustomJob.perform_later(repo.index.name, [id], **kwargs)
65
+ # end
66
+ def task(*operations, &block)
67
+ tasks.define(*operations, &block)
68
+ end
69
+
70
+ private
71
+
72
+ def bg_job_config
73
+ BackgroundJob.config
74
+ end
75
+
76
+ def default_jobs
77
+ Esse::AsyncIndexing::Jobs::DEFAULT.values
78
+ end
43
79
  end
44
80
  end
45
81
  end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ class Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob
4
+ def perform(index_class_name, repo_name, attribute_name, ids, options = {})
5
+ Esse::AsyncIndexing::Actions::BulkUpdateLazyAttribute.call(index_class_name, repo_name, attribute_name, ids, options)
6
+ end
7
+ end
@@ -2,6 +2,6 @@
2
2
 
3
3
  class Esse::AsyncIndexing::Jobs::ImportAllJob
4
4
  def perform(index_class_name, repo_name, options = {})
5
- Esse::AsyncIndexing::Actions::BatchImportAll.call(index_class_name, repo_name, options)
5
+ Esse::AsyncIndexing::Actions::BulkImportAll.call(index_class_name, repo_name, options)
6
6
  end
7
7
  end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ class Esse::AsyncIndexing::Jobs::ImportIdsJob
4
+ def perform(index_class_name, repo_name, ids, options = {})
5
+ # This is specific to the AsyncIndexing plugin, can't pass to Esse import method
6
+ enqueue_lazy = options.delete(:enqueue_lazy_attributes) if options.key?(:enqueue_lazy_attributes)
7
+ enqueue_lazy = options.delete("enqueue_lazy_attributes") if options.key?("enqueue_lazy_attributes")
8
+ enqueue_lazy = true if enqueue_lazy.nil?
9
+ total = Esse::AsyncIndexing::Actions::BulkImport.call(index_class_name, repo_name, ids, options)
10
+ options = Esse::HashUtils.deep_transform_keys(options, &:to_s)
11
+
12
+ return total if total.zero?
13
+ return total unless enqueue_lazy
14
+ return total if lazy_already_imported?(options)
15
+ return total unless self.class.respond_to?(:background_job_service)
16
+
17
+ index_class, repo_class = Esse::AsyncIndexing::Actions::CoerceIndexRepository.call(index_class_name, repo_name)
18
+ return total unless Esse::AsyncIndexing.plugin_installed?(index_class)
19
+
20
+ repo_class.lazy_document_attributes.each_key do |attr_name|
21
+ repo_class.async_indexing_job_for(:update_lazy_attribute).call(
22
+ **options,
23
+ service: self.class.background_job_service,
24
+ repo: repo_class,
25
+ operation: :update_lazy_attribute,
26
+ attribute: attr_name,
27
+ ids: ids
28
+ )
29
+ end
30
+ total
31
+ end
32
+
33
+ protected
34
+
35
+ # The `import` action already eager or lazy load the document attributes when some of these options are set.
36
+ def lazy_already_imported?(options)
37
+ preload = options.delete("preload_lazy_attributes") || false
38
+ eager = options.delete("eager_load_lazy_attributes") || false
39
+ lazy = options.delete("update_lazy_attributes") || false
40
+ preload || eager || lazy
41
+ end
42
+ end
@@ -1,20 +1,17 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require_relative "worker"
4
-
5
3
  module Esse
6
4
  module AsyncIndexing
7
- module Workers
5
+ module Jobs
8
6
  DEFAULT = {
9
- "esse/async_indexing/jobs/bulk_update_lazy_document_attribute_job" => "Esse::AsyncIndexing::Jobs::BulkUpdateLazyDocumentAttributeJob",
7
+ "esse/async_indexing/jobs/bulk_update_lazy_attribute_job" => "Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob",
10
8
  "esse/async_indexing/jobs/document_delete_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob",
11
9
  "esse/async_indexing/jobs/document_index_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob",
12
10
  "esse/async_indexing/jobs/document_update_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob",
13
11
  "esse/async_indexing/jobs/document_upsert_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentUpsertByIdJob",
14
12
  "esse/async_indexing/jobs/import_all_job" => "Esse::AsyncIndexing::Jobs::ImportAllJob",
15
- "esse/async_indexing/jobs/import_batch_id_job" => "Esse::AsyncIndexing::Jobs::ImportBatchIdJob",
16
- "esse/async_indexing/jobs/update_lazy_document_attribute_job" => "Esse::AsyncIndexing::Jobs::UpdateLazyDocumentAttributeJob"
17
- }
13
+ "esse/async_indexing/jobs/import_ids_job" => "Esse::AsyncIndexing::Jobs::ImportIdsJob"
14
+ }.freeze
18
15
 
19
16
  # The backend service may live in a different application, so they are not required by default.
20
17
  # That's why we have a separate structure to let enqueue jobs even without having the explicit worker class loaded.
@@ -22,27 +19,13 @@ module Esse
22
19
  def self.install!(service, **options)
23
20
  return if @installed_services&.include?(service.to_sym)
24
21
 
25
- DEFAULT.each do |job, worker_name|
22
+ DEFAULT.each do |job, const_name|
26
23
  Kernel.require(job)
27
- worker = Esse::AsyncIndexing::Jobs.const_get(worker_name.split("::").last)
28
- worker.extend(self.for(service, **options))
24
+ klass = Esse::AsyncIndexing::Jobs.const_get(const_name.split("::").last)
25
+ klass.extend BackgroundJob.mixin(service, **options)
29
26
  end
30
27
  @installed_services = Array(@installed_services) << service.to_sym
31
28
  end
32
-
33
- def self.for(service, **options)
34
- require_relative "workers/#{service}"
35
- service = service.to_sym
36
- worker_options = options.merge(service: service)
37
- module_name = service.to_s.split(/_/i).collect! { |w| w.capitalize }.join
38
- mod = Esse::AsyncIndexing::Workers.const_get(module_name)
39
- mod.module_eval do
40
- define_method(:bg_worker_options) do
41
- worker_options
42
- end
43
- end
44
- mod
45
- end
46
29
  end
47
30
  end
48
31
  end
@@ -0,0 +1,82 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Esse
4
+ module AsyncIndexing
5
+ class Tasks
6
+ DEFAULT = {
7
+ import: ->(service:, repo:, operation:, ids:, job_options: {}, **kwargs) {
8
+ unless (ids = Esse::ArrayUtils.wrap(ids)).empty?
9
+ BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::ImportIdsJob", **job_options)
10
+ .with_args(repo.index.name, repo.repo_name, ids, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
11
+ .push
12
+ end
13
+ },
14
+ index: ->(service:, repo:, operation:, id:, job_options: {}, **kwargs) {
15
+ if id
16
+ BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob", **job_options)
17
+ .with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
18
+ .push
19
+ end
20
+ },
21
+ update: ->(service:, repo:, operation:, id:, job_options: {}, **kwargs) {
22
+ if id
23
+ BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob", **job_options)
24
+ .with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
25
+ .push
26
+ end
27
+ },
28
+ delete: ->(service:, repo:, operation:, id:, job_options: {}, **kwargs) {
29
+ if id
30
+ BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob", **job_options)
31
+ .with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
32
+ .push
33
+ end
34
+ },
35
+ update_lazy_attribute: ->(service:, repo:, operation:, attribute:, ids:, job_options: {}, **kwargs) {
36
+ unless (ids = Esse::ArrayUtils.wrap(ids)).empty?
37
+ BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob", **job_options)
38
+ .with_args(repo.index.name, repo.repo_name, attribute.to_s, ids, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
39
+ .push
40
+ end
41
+ }
42
+ }.freeze
43
+
44
+ def initialize
45
+ @tasks = {}.freeze
46
+ end
47
+
48
+ def user_defined?(name)
49
+ @tasks.key?(name.to_sym)
50
+ end
51
+
52
+ def define(*names, &block)
53
+ names = DEFAULT.keys if names.empty?
54
+ validate!(names, block)
55
+ new_tasks = names.each_with_object({}) { |name, h| h[name.to_sym] = block }
56
+ @tasks = @tasks.dup.merge(new_tasks)
57
+ ensure
58
+ @tasks.freeze
59
+ end
60
+
61
+ def fetch(name)
62
+ id = name.to_sym
63
+ @tasks[id] || DEFAULT[id] || raise(ArgumentError, "Unknown task: #{name}")
64
+ end
65
+ alias_method :[], :fetch
66
+
67
+ private
68
+
69
+ def validate!(names, block)
70
+ unless block.is_a?(Proc)
71
+ raise ArgumentError, "The block of task must be a callable object"
72
+ end
73
+
74
+ names.each do |name|
75
+ unless DEFAULT.key?(name)
76
+ raise ArgumentError, "Unrecognized task: #{name}. Valid tasks are: #{DEFAULT.keys}"
77
+ end
78
+ end
79
+ end
80
+ end
81
+ end
82
+ end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Esse
4
4
  module AsyncIndexing
5
- VERSION = "0.0.2"
5
+ VERSION = "0.1.0.rc1"
6
6
  end
7
7
  end
@@ -1,11 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "esse"
4
- require "esse-redis_storage"
5
4
  require "forwardable"
6
5
  require "securerandom"
7
6
  require "time"
8
7
  require "multi_json"
8
+ require "background_job"
9
9
 
10
10
  module Esse
11
11
  module AsyncIndexing
@@ -14,32 +14,24 @@ module Esse
14
14
 
15
15
  module Jobs
16
16
  end
17
-
18
- module Workers
19
- end
20
17
  end
21
18
  end
22
19
 
23
20
  require_relative "async_indexing/version"
24
21
  require_relative "async_indexing/actions"
25
- require_relative "async_indexing/adapters"
26
22
  require_relative "async_indexing/config"
23
+ require_relative "async_indexing/tasks"
27
24
  require_relative "async_indexing/errors"
28
- require_relative "async_indexing/workers"
25
+ require_relative "async_indexing/jobs"
29
26
  require_relative "plugins/async_indexing"
30
27
 
31
28
  module Esse::AsyncIndexing
32
- SERVICES = {
33
- sidekiq: Adapters::Sidekiq,
34
- faktory: Adapters::Faktory
35
- }
36
-
37
29
  # @param worker_class [String] The worker class name
38
30
  # @param options [Hash] Options that will be passed along to the worker instance
39
31
  # @return [Esse::AsyncIndexing::Worker] An instance of worker
40
- def self.worker(worker_class, service: nil, **options)
32
+ def self.worker(worker_class, service:, **options)
41
33
  serv_name = service_name(service)
42
- Worker.new(worker_class, **Esse.config.async_indexing.send(service).worker_options(worker_class).merge(options), service: serv_name)
34
+ BackgroundJob.job(serv_name, worker_class, **options)
43
35
  end
44
36
 
45
37
  def self.service_name(identifier = nil)
@@ -47,26 +39,29 @@ module Esse::AsyncIndexing
47
39
  if identifier.nil?
48
40
  raise ArgumentError, "There are no async indexing services configured. Please configure at least one service or pass the service name as an argument."
49
41
  end
50
-
51
- if SERVICES[identifier.to_sym].nil?
52
- raise ArgumentError, "Invalid service: #{identifier.inspect}, valid services are: #{SERVICES.keys.join(", ")}"
42
+ unless (services = BackgroundJob::SERVICES).key?(identifier.to_sym)
43
+ raise ArgumentError, "Invalid service: #{identifier.inspect}, valid services are: #{services.keys.join(", ")}"
53
44
  end
54
45
 
55
46
  identifier.to_sym
56
47
  end
57
48
 
58
- def self.jid
59
- SecureRandom.hex(12)
60
- end
61
-
62
49
  def self.async_indexing_repo?(repo)
63
50
  return false unless repo.is_a?(Class) && repo < Esse::Repository
64
51
 
65
52
  repo.respond_to?(:implement_batch_ids?) && repo.implement_batch_ids?
66
53
  end
54
+
55
+ def self.plugin_installed?(index)
56
+ index = index.index if index.is_a?(Class) && index < Esse::Repository
57
+ return false unless index.is_a?(Class) && index < Esse::Index
58
+
59
+ index.plugins.include?(Esse::Plugins::AsyncIndexing)
60
+ end
67
61
  end
68
62
 
69
63
  Esse::Config.__send__ :include, Esse::AsyncIndexing::Config
64
+
70
65
  if defined?(Esse::CLI)
71
66
  require_relative "async_indexing/cli"
72
67
  end
@@ -4,48 +4,23 @@ module Esse
4
4
  module Plugins
5
5
  module AsyncIndexing
6
6
  module RepositoryClassMethods
7
- DEFAULT_ASYNC_INDEXING_JOBS = {
8
- import: ->(service:, repo:, operation:, ids:, **kwargs) {
9
- unless (ids = Esse::ArrayUtils.wrap(ids)).empty?
10
- batch_id = Esse::RedisStorage::Queue.for(repo: repo).enqueue(values: ids)
11
- Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::ImportBatchIdJob", service: service)
12
- .with_args(repo.index.name, repo.repo_name, batch_id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
13
- .push
14
- end
15
- },
16
- index: ->(service:, repo:, operation:, id:, **kwargs) {
17
- if id
18
- Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob", service: service)
19
- .with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
20
- .push
21
- end
22
- },
23
- update: ->(service:, repo:, operation:, id:, **kwargs) {
24
- if id
25
- Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob", service: service)
26
- .with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
27
- .push
28
- end
29
- },
30
- delete: ->(service:, repo:, operation:, id:, **kwargs) {
31
- if id
32
- Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob", service: service)
33
- .with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
34
- .push
35
- end
36
- }
37
- }.freeze
38
-
39
7
  # This method is used to retrieve only the ids of the documents in the collection.
40
8
  # It's used to asynchronously index the documents.
41
9
  # The #each_batch_ids method is optional and should be implemented by the collection class.
42
10
  #
11
+ # @yield [batch] Yields each batch of ids
43
12
  # @return [Enumerator] The enumerator
44
- def batch_ids(*args, **kwargs, &block)
13
+ def batch_ids(*args, **kwargs)
45
14
  if implement_batch_ids?
46
- Enumerator.new do |yielder|
15
+ if block_given?
47
16
  @collection_proc.new(*args, **kwargs).each_batch_ids do |batch|
48
- yielder.yield(batch)
17
+ yield(batch)
18
+ end
19
+ else
20
+ Enumerator.new do |yielder|
21
+ @collection_proc.new(*args, **kwargs).each_batch_ids do |batch|
22
+ yielder.yield(batch)
23
+ end
49
24
  end
50
25
  end
51
26
  else
@@ -70,35 +45,19 @@ module Esse
70
45
  # MyCustomJob.perform_later(repo.index.name, [id], **kwargs)
71
46
  # end
72
47
  def async_indexing_job(*operations, &block)
73
- operations = AsyncIndexingJobValidator::OPERATIONS if operations.empty?
74
- AsyncIndexingJobValidator.call(operations, block)
75
- hash = operations.each_with_object({}) { |operation, h| h[operation] = block }
76
- @async_indexing_jobs = async_indexing_jobs.dup.merge(hash)
77
- ensure
78
- @async_indexing_jobs.freeze
48
+ definer = @async_indexing_tasks || Esse::AsyncIndexing::Tasks.new
49
+ definer.define(*operations, &block)
50
+ @async_indexing_tasks = definer
79
51
  end
80
52
 
81
- def async_indexing_jobs
82
- @async_indexing_jobs || {}.freeze
83
- end
53
+ def async_indexing_job?(operation)
54
+ return false unless @async_indexing_tasks
84
55
 
85
- def async_indexing_job_for(operation)
86
- async_indexing_jobs[operation] || DEFAULT_ASYNC_INDEXING_JOBS[operation] || raise(ArgumentError, "The #{operation} operation is not implemented")
56
+ @async_indexing_tasks.user_defined?(operation)
87
57
  end
88
58
 
89
- class AsyncIndexingJobValidator
90
- OPERATIONS = %i[import index update delete].freeze
91
-
92
- def self.call(operations, block)
93
- unless block.is_a?(Proc)
94
- raise ArgumentError, "The block of async_indexing_job must be a callable object"
95
- end
96
-
97
- operations.each do |operation|
98
- next if OPERATIONS.include?(operation)
99
- raise ArgumentError, format("Unrecognized operation: %<operation>p. Valid operations are: %<valid>p", operation: operation, valid: OPERATIONS)
100
- end
101
- end
59
+ def async_indexing_job_for(operation)
60
+ (@async_indexing_tasks || Esse.config.async_indexing.tasks).fetch(operation)
102
61
  end
103
62
  end
104
63
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: esse-async_indexing
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.1.0.rc1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Marcos G. Zimmermann
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-08-02 00:00:00.000000000 Z
11
+ date: 2024-08-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: esse
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 0.3.4
19
+ version: 0.4.0.rc1
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: 0.3.4
26
+ version: 0.4.0.rc1
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: multi_json
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -39,19 +39,19 @@ dependencies:
39
39
  - !ruby/object:Gem::Version
40
40
  version: 0.0.0
41
41
  - !ruby/object:Gem::Dependency
42
- name: esse-redis_storage
42
+ name: background_job
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - ">="
46
46
  - !ruby/object:Gem::Version
47
- version: 0.1.0
47
+ version: 0.0.0
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
- version: 0.1.0
54
+ version: 0.0.0
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: pry
57
57
  requirement: !ruby/object:Gem::Requirement
@@ -169,17 +169,15 @@ files:
169
169
  - lib/esse-async_indexing.rb
170
170
  - lib/esse/async_indexing.rb
171
171
  - lib/esse/async_indexing/actions.rb
172
- - lib/esse/async_indexing/actions/batch_delete.rb
173
- - lib/esse/async_indexing/actions/batch_import.rb
174
- - lib/esse/async_indexing/actions/batch_import_all.rb
175
- - lib/esse/async_indexing/actions/batch_update.rb
176
- - lib/esse/async_indexing/actions/bulk_update_lazy_document_attribute.rb
172
+ - lib/esse/async_indexing/actions/bulk_delete.rb
173
+ - lib/esse/async_indexing/actions/bulk_import.rb
174
+ - lib/esse/async_indexing/actions/bulk_import_all.rb
175
+ - lib/esse/async_indexing/actions/bulk_update.rb
176
+ - lib/esse/async_indexing/actions/bulk_update_lazy_attribute.rb
177
177
  - lib/esse/async_indexing/actions/coerce_index_repository.rb
178
178
  - lib/esse/async_indexing/actions/delete_document.rb
179
- - lib/esse/async_indexing/actions/import_batch_id.rb
180
179
  - lib/esse/async_indexing/actions/index_document.rb
181
180
  - lib/esse/async_indexing/actions/update_document.rb
182
- - lib/esse/async_indexing/actions/update_lazy_document_attribute.rb
183
181
  - lib/esse/async_indexing/actions/upsert_document.rb
184
182
  - lib/esse/async_indexing/active_record.rb
185
183
  - lib/esse/async_indexing/active_record_callbacks/callback.rb
@@ -187,33 +185,22 @@ files:
187
185
  - lib/esse/async_indexing/active_record_callbacks/on_create.rb
188
186
  - lib/esse/async_indexing/active_record_callbacks/on_destroy.rb
189
187
  - lib/esse/async_indexing/active_record_callbacks/on_update.rb
190
- - lib/esse/async_indexing/adapters.rb
191
- - lib/esse/async_indexing/adapters/adapter.rb
192
- - lib/esse/async_indexing/adapters/faktory.rb
193
- - lib/esse/async_indexing/adapters/sidekiq.rb
194
188
  - lib/esse/async_indexing/cli.rb
195
189
  - lib/esse/async_indexing/cli/async_import.rb
190
+ - lib/esse/async_indexing/cli/async_update_lazy_attributes.rb
196
191
  - lib/esse/async_indexing/config.rb
197
192
  - lib/esse/async_indexing/configuration.rb
198
- - lib/esse/async_indexing/configuration/base.rb
199
- - lib/esse/async_indexing/configuration/faktory.rb
200
- - lib/esse/async_indexing/configuration/sidekiq.rb
201
193
  - lib/esse/async_indexing/errors.rb
202
- - lib/esse/async_indexing/jobs/bulk_update_lazy_document_attribute_job.rb
194
+ - lib/esse/async_indexing/jobs.rb
195
+ - lib/esse/async_indexing/jobs/bulk_update_lazy_attribute_job.rb
203
196
  - lib/esse/async_indexing/jobs/document_delete_by_id_job.rb
204
197
  - lib/esse/async_indexing/jobs/document_index_by_id_job.rb
205
198
  - lib/esse/async_indexing/jobs/document_update_by_id_job.rb
206
199
  - lib/esse/async_indexing/jobs/document_upsert_by_id_job.rb
207
200
  - lib/esse/async_indexing/jobs/import_all_job.rb
208
- - lib/esse/async_indexing/jobs/import_batch_id_job.rb
209
- - lib/esse/async_indexing/jobs/update_lazy_document_attribute_job.rb
210
- - lib/esse/async_indexing/testing.rb
201
+ - lib/esse/async_indexing/jobs/import_ids_job.rb
202
+ - lib/esse/async_indexing/tasks.rb
211
203
  - lib/esse/async_indexing/version.rb
212
- - lib/esse/async_indexing/worker.rb
213
- - lib/esse/async_indexing/workers.rb
214
- - lib/esse/async_indexing/workers/faktory.rb
215
- - lib/esse/async_indexing/workers/shared_class_methods.rb
216
- - lib/esse/async_indexing/workers/sidekiq.rb
217
204
  - lib/esse/plugins/async_indexing.rb
218
205
  homepage: https://github.com/marcosgz/esse-async_indexing
219
206
  licenses:
@@ -233,9 +220,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
233
220
  version: 2.7.0
234
221
  required_rubygems_version: !ruby/object:Gem::Requirement
235
222
  requirements:
236
- - - ">="
223
+ - - ">"
237
224
  - !ruby/object:Gem::Version
238
- version: '0'
225
+ version: 1.3.1
239
226
  requirements: []
240
227
  rubygems_version: 3.2.33
241
228
  signing_key: