esse-async_indexing 0.0.2 → 0.1.0.rc2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/Gemfile +2 -1
- data/Gemfile.lock +18 -12
- data/README.md +50 -41
- data/lib/esse/async_indexing/actions/{batch_delete.rb → bulk_delete.rb} +2 -2
- data/lib/esse/async_indexing/actions/{batch_import.rb → bulk_import.rb} +1 -1
- data/lib/esse/async_indexing/actions/{batch_import_all.rb → bulk_import_all.rb} +1 -1
- data/lib/esse/async_indexing/actions/{batch_update.rb → bulk_update.rb} +6 -4
- data/lib/esse/async_indexing/actions/bulk_update_lazy_attribute.rb +19 -0
- data/lib/esse/async_indexing/actions/delete_document.rb +1 -1
- data/lib/esse/async_indexing/actions/index_document.rb +1 -1
- data/lib/esse/async_indexing/actions/update_document.rb +1 -1
- data/lib/esse/async_indexing/actions/upsert_document.rb +1 -2
- data/lib/esse/async_indexing/actions.rb +5 -7
- data/lib/esse/async_indexing/active_record_callbacks/lazy_update_attribute.rb +8 -5
- data/lib/esse/async_indexing/cli/async_import.rb +27 -18
- data/lib/esse/async_indexing/cli/async_update_lazy_attributes.rb +78 -0
- data/lib/esse/async_indexing/cli.rb +23 -9
- data/lib/esse/async_indexing/config.rb +0 -3
- data/lib/esse/async_indexing/configuration.rb +54 -18
- data/lib/esse/async_indexing/jobs/bulk_update_lazy_attribute_job.rb +7 -0
- data/lib/esse/async_indexing/jobs/import_all_job.rb +1 -1
- data/lib/esse/async_indexing/jobs/import_ids_job.rb +42 -0
- data/lib/esse/async_indexing/{workers.rb → jobs.rb} +7 -24
- data/lib/esse/async_indexing/tasks.rb +82 -0
- data/lib/esse/async_indexing/version.rb +1 -1
- data/lib/esse/async_indexing.rb +15 -20
- data/lib/esse/plugins/async_indexing.rb +18 -59
- metadata +19 -32
- data/lib/esse/async_indexing/actions/bulk_update_lazy_document_attribute.rb +0 -20
- data/lib/esse/async_indexing/actions/import_batch_id.rb +0 -21
- data/lib/esse/async_indexing/actions/update_lazy_document_attribute.rb +0 -14
- data/lib/esse/async_indexing/adapters/adapter.rb +0 -29
- data/lib/esse/async_indexing/adapters/faktory.rb +0 -114
- data/lib/esse/async_indexing/adapters/sidekiq.rb +0 -94
- data/lib/esse/async_indexing/adapters.rb +0 -12
- data/lib/esse/async_indexing/configuration/base.rb +0 -65
- data/lib/esse/async_indexing/configuration/faktory.rb +0 -6
- data/lib/esse/async_indexing/configuration/sidekiq.rb +0 -12
- data/lib/esse/async_indexing/jobs/bulk_update_lazy_document_attribute_job.rb +0 -7
- data/lib/esse/async_indexing/jobs/import_batch_id_job.rb +0 -34
- data/lib/esse/async_indexing/jobs/update_lazy_document_attribute_job.rb +0 -7
- data/lib/esse/async_indexing/testing.rb +0 -79
- data/lib/esse/async_indexing/worker.rb +0 -85
- data/lib/esse/async_indexing/workers/faktory.rb +0 -28
- data/lib/esse/async_indexing/workers/shared_class_methods.rb +0 -26
- data/lib/esse/async_indexing/workers/sidekiq.rb +0 -28
@@ -13,20 +13,34 @@ Esse::CLI::Index.class_eval do
|
|
13
13
|
option :suffix, type: :string, default: nil, aliases: "-s", desc: "Suffix to append to index name"
|
14
14
|
option :context, type: :hash, default: {}, required: true, desc: "List of options to pass to the index class"
|
15
15
|
option :service, type: :string, default: nil, alias: "-s", desc: "Service to use for async import: sidekiq, faktory"
|
16
|
-
option :
|
17
|
-
option :
|
16
|
+
option :preload_lazy_attributes, type: :string, default: nil, desc: "Command separated list of lazy document attributes to preload using search API before the bulk import. Or pass `true` to preload all lazy attributes"
|
17
|
+
option :eager_load_lazy_attributes, type: :string, default: nil, desc: "Comma separated list of lazy document attributes to include to the bulk index request. Or pass `true` to include all lazy attributes"
|
18
|
+
option :update_lazy_attributes, type: :string, default: nil, desc: "Comma separated list of lazy document attributes to bulk update after the bulk index request Or pass `true` to include all lazy attributes"
|
19
|
+
option :enqueue_lazy_attributes, type: :boolean, default: nil, desc: "Enqueue the lazy document attributes job after the bulk import. (default: true))"
|
20
|
+
option :job_options, type: :hash, default: {}, desc: "List of options to pass to the background job. (Example: --job-options=queue:default)"
|
18
21
|
def async_import(*index_classes)
|
19
22
|
opts = Esse::HashUtils.deep_transform_keys(options.to_h, &:to_sym)
|
20
23
|
opts[:service] ||= Esse.config.async_indexing.services.first
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
end
|
26
|
-
if (val = opts[:lazy_update_document_attributes])
|
27
|
-
opts[:lazy_update_document_attributes] = (val == "true") ? true : val.split(",")
|
24
|
+
%i[preload_lazy_attributes eager_load_lazy_attributes update_lazy_attributes].each do |key|
|
25
|
+
if (val = opts.delete(key)) && val != "false"
|
26
|
+
opts[key] = (val == "true") ? true : val.split(",")
|
27
|
+
end
|
28
28
|
end
|
29
29
|
require "esse/async_indexing/cli/async_import"
|
30
30
|
Esse::AsyncIndexing::CLI::AsyncImport.new(indices: index_classes, **opts).run
|
31
31
|
end
|
32
|
+
|
33
|
+
desc "async_update_lazy_attributes INDEX_CLASS", "Async update lazy attributes for the given index"
|
34
|
+
option :repo, type: :string, default: nil, alias: "-r", desc: "Repository to use for import"
|
35
|
+
option :suffix, type: :string, default: nil, aliases: "-s", desc: "Suffix to append to index name"
|
36
|
+
option :context, type: :hash, default: {}, required: true, desc: "List of options to pass to the index class"
|
37
|
+
option :service, type: :string, default: nil, alias: "-s", desc: "Service to use for async import: sidekiq, faktory"
|
38
|
+
option :job_options, type: :hash, default: {}, desc: "List of options to pass to the background job. (Example: --job-options=queue:default)"
|
39
|
+
def async_update_lazy_attributes(index_class, *attributes)
|
40
|
+
opts = Esse::HashUtils.deep_transform_keys(options.to_h, &:to_sym)
|
41
|
+
opts[:service] ||= Esse.config.async_indexing.services.first
|
42
|
+
require "esse/async_indexing/cli/async_update_lazy_attributes"
|
43
|
+
|
44
|
+
Esse::AsyncIndexing::CLI::AsyncUpdateLazyAttributes.new(indices: [index_class], attributes: attributes, **opts).run
|
45
|
+
end
|
32
46
|
end
|
@@ -17,29 +17,65 @@ module Esse
|
|
17
17
|
@services ||= ConfigService.new
|
18
18
|
end
|
19
19
|
|
20
|
-
def
|
21
|
-
@
|
22
|
-
|
23
|
-
|
24
|
-
end
|
25
|
-
if block_given?
|
26
|
-
yield @faktory
|
27
|
-
else
|
28
|
-
@faktory
|
29
|
-
end
|
20
|
+
def reset!
|
21
|
+
@services = nil
|
22
|
+
@tasks = nil
|
23
|
+
bg_job_config.reset!
|
30
24
|
end
|
31
25
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
Configuration::Sidekiq.new
|
26
|
+
BackgroundJob::SERVICES.each_key do |service|
|
27
|
+
define_method(service) do |&block|
|
28
|
+
config_for(service, &block)
|
36
29
|
end
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
30
|
+
end
|
31
|
+
|
32
|
+
def config_for(service)
|
33
|
+
case service.to_sym
|
34
|
+
when :faktory
|
35
|
+
conf = bg_job_config.faktory
|
36
|
+
unless services.faktory?
|
37
|
+
default_jobs.each { |job_class| conf.jobs[job_class] ||= {} }
|
38
|
+
services << :faktory
|
39
|
+
end
|
40
|
+
yield conf if block_given?
|
41
|
+
conf
|
42
|
+
when :sidekiq
|
43
|
+
conf = bg_job_config.sidekiq
|
44
|
+
unless services.sidekiq?
|
45
|
+
default_jobs.each { |job_class| conf.jobs[job_class] ||= {} }
|
46
|
+
services << :sidekiq
|
47
|
+
end
|
48
|
+
yield conf if block_given?
|
49
|
+
conf
|
50
|
+
else raise ArgumentError, "Unknown service: #{service}"
|
41
51
|
end
|
42
52
|
end
|
53
|
+
|
54
|
+
def tasks
|
55
|
+
@tasks ||= Esse::AsyncIndexing::Tasks.new
|
56
|
+
end
|
57
|
+
|
58
|
+
# DSL to define custom job enqueueing
|
59
|
+
#
|
60
|
+
# task(:import) do |service:, repo:, operation:, ids:, **kwargs|
|
61
|
+
# MyCustomJob.perform_later(repo.index.name, ids, **kwargs)
|
62
|
+
# end
|
63
|
+
# task(:index, :update, :delete) do |service:, repo:, operation:, id, **kwargs|
|
64
|
+
# MyCustomJob.perform_later(repo.index.name, [id], **kwargs)
|
65
|
+
# end
|
66
|
+
def task(*operations, &block)
|
67
|
+
tasks.define(*operations, &block)
|
68
|
+
end
|
69
|
+
|
70
|
+
private
|
71
|
+
|
72
|
+
def bg_job_config
|
73
|
+
BackgroundJob.config
|
74
|
+
end
|
75
|
+
|
76
|
+
def default_jobs
|
77
|
+
Esse::AsyncIndexing::Jobs::DEFAULT.values
|
78
|
+
end
|
43
79
|
end
|
44
80
|
end
|
45
81
|
end
|
@@ -0,0 +1,7 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob
|
4
|
+
def perform(index_class_name, repo_name, attribute_name, ids, options = {})
|
5
|
+
Esse::AsyncIndexing::Actions::BulkUpdateLazyAttribute.call(index_class_name, repo_name, attribute_name, ids, options)
|
6
|
+
end
|
7
|
+
end
|
@@ -2,6 +2,6 @@
|
|
2
2
|
|
3
3
|
class Esse::AsyncIndexing::Jobs::ImportAllJob
|
4
4
|
def perform(index_class_name, repo_name, options = {})
|
5
|
-
Esse::AsyncIndexing::Actions::
|
5
|
+
Esse::AsyncIndexing::Actions::BulkImportAll.call(index_class_name, repo_name, options)
|
6
6
|
end
|
7
7
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Esse::AsyncIndexing::Jobs::ImportIdsJob
|
4
|
+
def perform(index_class_name, repo_name, ids, options = {})
|
5
|
+
# This is specific to the AsyncIndexing plugin, can't pass to Esse import method
|
6
|
+
enqueue_lazy = options.delete(:enqueue_lazy_attributes) if options.key?(:enqueue_lazy_attributes)
|
7
|
+
enqueue_lazy = options.delete("enqueue_lazy_attributes") if options.key?("enqueue_lazy_attributes")
|
8
|
+
enqueue_lazy = true if enqueue_lazy.nil?
|
9
|
+
total = Esse::AsyncIndexing::Actions::BulkImport.call(index_class_name, repo_name, ids, options)
|
10
|
+
options = Esse::HashUtils.deep_transform_keys(options, &:to_s)
|
11
|
+
|
12
|
+
return total if total.zero?
|
13
|
+
return total unless enqueue_lazy
|
14
|
+
return total if lazy_already_imported?(options)
|
15
|
+
return total unless self.class.respond_to?(:background_job_service)
|
16
|
+
|
17
|
+
index_class, repo_class = Esse::AsyncIndexing::Actions::CoerceIndexRepository.call(index_class_name, repo_name)
|
18
|
+
return total unless Esse::AsyncIndexing.plugin_installed?(index_class)
|
19
|
+
|
20
|
+
repo_class.lazy_document_attributes.each_key do |attr_name|
|
21
|
+
repo_class.async_indexing_job_for(:update_lazy_attribute).call(
|
22
|
+
**options,
|
23
|
+
service: self.class.background_job_service,
|
24
|
+
repo: repo_class,
|
25
|
+
operation: :update_lazy_attribute,
|
26
|
+
attribute: attr_name,
|
27
|
+
ids: ids
|
28
|
+
)
|
29
|
+
end
|
30
|
+
total
|
31
|
+
end
|
32
|
+
|
33
|
+
protected
|
34
|
+
|
35
|
+
# The `import` action already eager or lazy load the document attributes when some of these options are set.
|
36
|
+
def lazy_already_imported?(options)
|
37
|
+
preload = options.delete("preload_lazy_attributes") || false
|
38
|
+
eager = options.delete("eager_load_lazy_attributes") || false
|
39
|
+
lazy = options.delete("update_lazy_attributes") || false
|
40
|
+
preload || eager || lazy
|
41
|
+
end
|
42
|
+
end
|
@@ -1,20 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require_relative "worker"
|
4
|
-
|
5
3
|
module Esse
|
6
4
|
module AsyncIndexing
|
7
|
-
module
|
5
|
+
module Jobs
|
8
6
|
DEFAULT = {
|
9
|
-
"esse/async_indexing/jobs/
|
7
|
+
"esse/async_indexing/jobs/bulk_update_lazy_attribute_job" => "Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob",
|
10
8
|
"esse/async_indexing/jobs/document_delete_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob",
|
11
9
|
"esse/async_indexing/jobs/document_index_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob",
|
12
10
|
"esse/async_indexing/jobs/document_update_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob",
|
13
11
|
"esse/async_indexing/jobs/document_upsert_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentUpsertByIdJob",
|
14
12
|
"esse/async_indexing/jobs/import_all_job" => "Esse::AsyncIndexing::Jobs::ImportAllJob",
|
15
|
-
"esse/async_indexing/jobs/
|
16
|
-
|
17
|
-
}
|
13
|
+
"esse/async_indexing/jobs/import_ids_job" => "Esse::AsyncIndexing::Jobs::ImportIdsJob"
|
14
|
+
}.freeze
|
18
15
|
|
19
16
|
# The backend service may live in a different application, so they are not required by default.
|
20
17
|
# That's why we have a separate structure to let enqueue jobs even without having the explicit worker class loaded.
|
@@ -22,27 +19,13 @@ module Esse
|
|
22
19
|
def self.install!(service, **options)
|
23
20
|
return if @installed_services&.include?(service.to_sym)
|
24
21
|
|
25
|
-
DEFAULT.each do |job,
|
22
|
+
DEFAULT.each do |job, const_name|
|
26
23
|
Kernel.require(job)
|
27
|
-
|
28
|
-
|
24
|
+
klass = Esse::AsyncIndexing::Jobs.const_get(const_name.split("::").last)
|
25
|
+
klass.extend BackgroundJob.mixin(service, **options)
|
29
26
|
end
|
30
27
|
@installed_services = Array(@installed_services) << service.to_sym
|
31
28
|
end
|
32
|
-
|
33
|
-
def self.for(service, **options)
|
34
|
-
require_relative "workers/#{service}"
|
35
|
-
service = service.to_sym
|
36
|
-
worker_options = options.merge(service: service)
|
37
|
-
module_name = service.to_s.split(/_/i).collect! { |w| w.capitalize }.join
|
38
|
-
mod = Esse::AsyncIndexing::Workers.const_get(module_name)
|
39
|
-
mod.module_eval do
|
40
|
-
define_method(:bg_worker_options) do
|
41
|
-
worker_options
|
42
|
-
end
|
43
|
-
end
|
44
|
-
mod
|
45
|
-
end
|
46
29
|
end
|
47
30
|
end
|
48
31
|
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Esse
|
4
|
+
module AsyncIndexing
|
5
|
+
class Tasks
|
6
|
+
DEFAULT = {
|
7
|
+
import: ->(service:, repo:, operation:, ids:, job_options: {}, **kwargs) {
|
8
|
+
unless (ids = Esse::ArrayUtils.wrap(ids)).empty?
|
9
|
+
BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::ImportIdsJob", **job_options)
|
10
|
+
.with_args(repo.index.name, repo.repo_name, ids, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
11
|
+
.push
|
12
|
+
end
|
13
|
+
},
|
14
|
+
index: ->(service:, repo:, operation:, id:, job_options: {}, **kwargs) {
|
15
|
+
if id
|
16
|
+
BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob", **job_options)
|
17
|
+
.with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
18
|
+
.push
|
19
|
+
end
|
20
|
+
},
|
21
|
+
update: ->(service:, repo:, operation:, id:, job_options: {}, **kwargs) {
|
22
|
+
if id
|
23
|
+
BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob", **job_options)
|
24
|
+
.with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
25
|
+
.push
|
26
|
+
end
|
27
|
+
},
|
28
|
+
delete: ->(service:, repo:, operation:, id:, job_options: {}, **kwargs) {
|
29
|
+
if id
|
30
|
+
BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob", **job_options)
|
31
|
+
.with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
32
|
+
.push
|
33
|
+
end
|
34
|
+
},
|
35
|
+
update_lazy_attribute: ->(service:, repo:, operation:, attribute:, ids:, job_options: {}, **kwargs) {
|
36
|
+
unless (ids = Esse::ArrayUtils.wrap(ids)).empty?
|
37
|
+
BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob", **job_options)
|
38
|
+
.with_args(repo.index.name, repo.repo_name, attribute.to_s, ids, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
39
|
+
.push
|
40
|
+
end
|
41
|
+
}
|
42
|
+
}.freeze
|
43
|
+
|
44
|
+
def initialize
|
45
|
+
@tasks = {}.freeze
|
46
|
+
end
|
47
|
+
|
48
|
+
def user_defined?(name)
|
49
|
+
@tasks.key?(name.to_sym)
|
50
|
+
end
|
51
|
+
|
52
|
+
def define(*names, &block)
|
53
|
+
names = DEFAULT.keys if names.empty?
|
54
|
+
validate!(names, block)
|
55
|
+
new_tasks = names.each_with_object({}) { |name, h| h[name.to_sym] = block }
|
56
|
+
@tasks = @tasks.dup.merge(new_tasks)
|
57
|
+
ensure
|
58
|
+
@tasks.freeze
|
59
|
+
end
|
60
|
+
|
61
|
+
def fetch(name)
|
62
|
+
id = name.to_sym
|
63
|
+
@tasks[id] || DEFAULT[id] || raise(ArgumentError, "Unknown task: #{name}")
|
64
|
+
end
|
65
|
+
alias_method :[], :fetch
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
def validate!(names, block)
|
70
|
+
unless block.is_a?(Proc)
|
71
|
+
raise ArgumentError, "The block of task must be a callable object"
|
72
|
+
end
|
73
|
+
|
74
|
+
names.each do |name|
|
75
|
+
unless DEFAULT.key?(name)
|
76
|
+
raise ArgumentError, "Unrecognized task: #{name}. Valid tasks are: #{DEFAULT.keys}"
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
data/lib/esse/async_indexing.rb
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "esse"
|
4
|
-
require "esse-redis_storage"
|
5
4
|
require "forwardable"
|
6
5
|
require "securerandom"
|
7
6
|
require "time"
|
8
7
|
require "multi_json"
|
8
|
+
require "background_job"
|
9
9
|
|
10
10
|
module Esse
|
11
11
|
module AsyncIndexing
|
@@ -14,32 +14,24 @@ module Esse
|
|
14
14
|
|
15
15
|
module Jobs
|
16
16
|
end
|
17
|
-
|
18
|
-
module Workers
|
19
|
-
end
|
20
17
|
end
|
21
18
|
end
|
22
19
|
|
23
20
|
require_relative "async_indexing/version"
|
24
21
|
require_relative "async_indexing/actions"
|
25
|
-
require_relative "async_indexing/adapters"
|
26
22
|
require_relative "async_indexing/config"
|
23
|
+
require_relative "async_indexing/tasks"
|
27
24
|
require_relative "async_indexing/errors"
|
28
|
-
require_relative "async_indexing/
|
25
|
+
require_relative "async_indexing/jobs"
|
29
26
|
require_relative "plugins/async_indexing"
|
30
27
|
|
31
28
|
module Esse::AsyncIndexing
|
32
|
-
SERVICES = {
|
33
|
-
sidekiq: Adapters::Sidekiq,
|
34
|
-
faktory: Adapters::Faktory
|
35
|
-
}
|
36
|
-
|
37
29
|
# @param worker_class [String] The worker class name
|
38
30
|
# @param options [Hash] Options that will be passed along to the worker instance
|
39
31
|
# @return [Esse::AsyncIndexing::Worker] An instance of worker
|
40
|
-
def self.worker(worker_class, service
|
32
|
+
def self.worker(worker_class, service:, **options)
|
41
33
|
serv_name = service_name(service)
|
42
|
-
|
34
|
+
BackgroundJob.job(serv_name, worker_class, **options)
|
43
35
|
end
|
44
36
|
|
45
37
|
def self.service_name(identifier = nil)
|
@@ -47,26 +39,29 @@ module Esse::AsyncIndexing
|
|
47
39
|
if identifier.nil?
|
48
40
|
raise ArgumentError, "There are no async indexing services configured. Please configure at least one service or pass the service name as an argument."
|
49
41
|
end
|
50
|
-
|
51
|
-
|
52
|
-
raise ArgumentError, "Invalid service: #{identifier.inspect}, valid services are: #{SERVICES.keys.join(", ")}"
|
42
|
+
unless (services = BackgroundJob::SERVICES).key?(identifier.to_sym)
|
43
|
+
raise ArgumentError, "Invalid service: #{identifier.inspect}, valid services are: #{services.keys.join(", ")}"
|
53
44
|
end
|
54
45
|
|
55
46
|
identifier.to_sym
|
56
47
|
end
|
57
48
|
|
58
|
-
def self.jid
|
59
|
-
SecureRandom.hex(12)
|
60
|
-
end
|
61
|
-
|
62
49
|
def self.async_indexing_repo?(repo)
|
63
50
|
return false unless repo.is_a?(Class) && repo < Esse::Repository
|
64
51
|
|
65
52
|
repo.respond_to?(:implement_batch_ids?) && repo.implement_batch_ids?
|
66
53
|
end
|
54
|
+
|
55
|
+
def self.plugin_installed?(index)
|
56
|
+
index = index.index if index.is_a?(Class) && index < Esse::Repository
|
57
|
+
return false unless index.is_a?(Class) && index < Esse::Index
|
58
|
+
|
59
|
+
index.plugins.include?(Esse::Plugins::AsyncIndexing)
|
60
|
+
end
|
67
61
|
end
|
68
62
|
|
69
63
|
Esse::Config.__send__ :include, Esse::AsyncIndexing::Config
|
64
|
+
|
70
65
|
if defined?(Esse::CLI)
|
71
66
|
require_relative "async_indexing/cli"
|
72
67
|
end
|
@@ -4,48 +4,23 @@ module Esse
|
|
4
4
|
module Plugins
|
5
5
|
module AsyncIndexing
|
6
6
|
module RepositoryClassMethods
|
7
|
-
DEFAULT_ASYNC_INDEXING_JOBS = {
|
8
|
-
import: ->(service:, repo:, operation:, ids:, **kwargs) {
|
9
|
-
unless (ids = Esse::ArrayUtils.wrap(ids)).empty?
|
10
|
-
batch_id = Esse::RedisStorage::Queue.for(repo: repo).enqueue(values: ids)
|
11
|
-
Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::ImportBatchIdJob", service: service)
|
12
|
-
.with_args(repo.index.name, repo.repo_name, batch_id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
13
|
-
.push
|
14
|
-
end
|
15
|
-
},
|
16
|
-
index: ->(service:, repo:, operation:, id:, **kwargs) {
|
17
|
-
if id
|
18
|
-
Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob", service: service)
|
19
|
-
.with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
20
|
-
.push
|
21
|
-
end
|
22
|
-
},
|
23
|
-
update: ->(service:, repo:, operation:, id:, **kwargs) {
|
24
|
-
if id
|
25
|
-
Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob", service: service)
|
26
|
-
.with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
27
|
-
.push
|
28
|
-
end
|
29
|
-
},
|
30
|
-
delete: ->(service:, repo:, operation:, id:, **kwargs) {
|
31
|
-
if id
|
32
|
-
Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob", service: service)
|
33
|
-
.with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
34
|
-
.push
|
35
|
-
end
|
36
|
-
}
|
37
|
-
}.freeze
|
38
|
-
|
39
7
|
# This method is used to retrieve only the ids of the documents in the collection.
|
40
8
|
# It's used to asynchronously index the documents.
|
41
9
|
# The #each_batch_ids method is optional and should be implemented by the collection class.
|
42
10
|
#
|
11
|
+
# @yield [batch] Yields each batch of ids
|
43
12
|
# @return [Enumerator] The enumerator
|
44
|
-
def batch_ids(*args, **kwargs
|
13
|
+
def batch_ids(*args, **kwargs)
|
45
14
|
if implement_batch_ids?
|
46
|
-
|
15
|
+
if block_given?
|
47
16
|
@collection_proc.new(*args, **kwargs).each_batch_ids do |batch|
|
48
|
-
|
17
|
+
yield(batch)
|
18
|
+
end
|
19
|
+
else
|
20
|
+
Enumerator.new do |yielder|
|
21
|
+
@collection_proc.new(*args, **kwargs).each_batch_ids do |batch|
|
22
|
+
yielder.yield(batch)
|
23
|
+
end
|
49
24
|
end
|
50
25
|
end
|
51
26
|
else
|
@@ -70,35 +45,19 @@ module Esse
|
|
70
45
|
# MyCustomJob.perform_later(repo.index.name, [id], **kwargs)
|
71
46
|
# end
|
72
47
|
def async_indexing_job(*operations, &block)
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
@async_indexing_jobs = async_indexing_jobs.dup.merge(hash)
|
77
|
-
ensure
|
78
|
-
@async_indexing_jobs.freeze
|
48
|
+
definer = @async_indexing_tasks || Esse::AsyncIndexing::Tasks.new
|
49
|
+
definer.define(*operations, &block)
|
50
|
+
@async_indexing_tasks = definer
|
79
51
|
end
|
80
52
|
|
81
|
-
def
|
82
|
-
|
83
|
-
end
|
53
|
+
def async_indexing_job?(operation)
|
54
|
+
return false unless @async_indexing_tasks
|
84
55
|
|
85
|
-
|
86
|
-
async_indexing_jobs[operation] || DEFAULT_ASYNC_INDEXING_JOBS[operation] || raise(ArgumentError, "The #{operation} operation is not implemented")
|
56
|
+
@async_indexing_tasks.user_defined?(operation)
|
87
57
|
end
|
88
58
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
def self.call(operations, block)
|
93
|
-
unless block.is_a?(Proc)
|
94
|
-
raise ArgumentError, "The block of async_indexing_job must be a callable object"
|
95
|
-
end
|
96
|
-
|
97
|
-
operations.each do |operation|
|
98
|
-
next if OPERATIONS.include?(operation)
|
99
|
-
raise ArgumentError, format("Unrecognized operation: %<operation>p. Valid operations are: %<valid>p", operation: operation, valid: OPERATIONS)
|
100
|
-
end
|
101
|
-
end
|
59
|
+
def async_indexing_job_for(operation)
|
60
|
+
(@async_indexing_tasks || Esse.config.async_indexing.tasks).fetch(operation)
|
102
61
|
end
|
103
62
|
end
|
104
63
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: esse-async_indexing
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.1.0.rc2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Marcos G. Zimmermann
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-08-
|
11
|
+
date: 2024-08-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: esse
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.
|
19
|
+
version: 0.4.0.rc1
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0.
|
26
|
+
version: 0.4.0.rc1
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: multi_json
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -39,19 +39,19 @@ dependencies:
|
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: 0.0.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: background_job
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.
|
47
|
+
version: 0.0.0
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.
|
54
|
+
version: 0.0.0
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: pry
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -169,17 +169,15 @@ files:
|
|
169
169
|
- lib/esse-async_indexing.rb
|
170
170
|
- lib/esse/async_indexing.rb
|
171
171
|
- lib/esse/async_indexing/actions.rb
|
172
|
-
- lib/esse/async_indexing/actions/
|
173
|
-
- lib/esse/async_indexing/actions/
|
174
|
-
- lib/esse/async_indexing/actions/
|
175
|
-
- lib/esse/async_indexing/actions/
|
176
|
-
- lib/esse/async_indexing/actions/
|
172
|
+
- lib/esse/async_indexing/actions/bulk_delete.rb
|
173
|
+
- lib/esse/async_indexing/actions/bulk_import.rb
|
174
|
+
- lib/esse/async_indexing/actions/bulk_import_all.rb
|
175
|
+
- lib/esse/async_indexing/actions/bulk_update.rb
|
176
|
+
- lib/esse/async_indexing/actions/bulk_update_lazy_attribute.rb
|
177
177
|
- lib/esse/async_indexing/actions/coerce_index_repository.rb
|
178
178
|
- lib/esse/async_indexing/actions/delete_document.rb
|
179
|
-
- lib/esse/async_indexing/actions/import_batch_id.rb
|
180
179
|
- lib/esse/async_indexing/actions/index_document.rb
|
181
180
|
- lib/esse/async_indexing/actions/update_document.rb
|
182
|
-
- lib/esse/async_indexing/actions/update_lazy_document_attribute.rb
|
183
181
|
- lib/esse/async_indexing/actions/upsert_document.rb
|
184
182
|
- lib/esse/async_indexing/active_record.rb
|
185
183
|
- lib/esse/async_indexing/active_record_callbacks/callback.rb
|
@@ -187,33 +185,22 @@ files:
|
|
187
185
|
- lib/esse/async_indexing/active_record_callbacks/on_create.rb
|
188
186
|
- lib/esse/async_indexing/active_record_callbacks/on_destroy.rb
|
189
187
|
- lib/esse/async_indexing/active_record_callbacks/on_update.rb
|
190
|
-
- lib/esse/async_indexing/adapters.rb
|
191
|
-
- lib/esse/async_indexing/adapters/adapter.rb
|
192
|
-
- lib/esse/async_indexing/adapters/faktory.rb
|
193
|
-
- lib/esse/async_indexing/adapters/sidekiq.rb
|
194
188
|
- lib/esse/async_indexing/cli.rb
|
195
189
|
- lib/esse/async_indexing/cli/async_import.rb
|
190
|
+
- lib/esse/async_indexing/cli/async_update_lazy_attributes.rb
|
196
191
|
- lib/esse/async_indexing/config.rb
|
197
192
|
- lib/esse/async_indexing/configuration.rb
|
198
|
-
- lib/esse/async_indexing/configuration/base.rb
|
199
|
-
- lib/esse/async_indexing/configuration/faktory.rb
|
200
|
-
- lib/esse/async_indexing/configuration/sidekiq.rb
|
201
193
|
- lib/esse/async_indexing/errors.rb
|
202
|
-
- lib/esse/async_indexing/jobs
|
194
|
+
- lib/esse/async_indexing/jobs.rb
|
195
|
+
- lib/esse/async_indexing/jobs/bulk_update_lazy_attribute_job.rb
|
203
196
|
- lib/esse/async_indexing/jobs/document_delete_by_id_job.rb
|
204
197
|
- lib/esse/async_indexing/jobs/document_index_by_id_job.rb
|
205
198
|
- lib/esse/async_indexing/jobs/document_update_by_id_job.rb
|
206
199
|
- lib/esse/async_indexing/jobs/document_upsert_by_id_job.rb
|
207
200
|
- lib/esse/async_indexing/jobs/import_all_job.rb
|
208
|
-
- lib/esse/async_indexing/jobs/
|
209
|
-
- lib/esse/async_indexing/
|
210
|
-
- lib/esse/async_indexing/testing.rb
|
201
|
+
- lib/esse/async_indexing/jobs/import_ids_job.rb
|
202
|
+
- lib/esse/async_indexing/tasks.rb
|
211
203
|
- lib/esse/async_indexing/version.rb
|
212
|
-
- lib/esse/async_indexing/worker.rb
|
213
|
-
- lib/esse/async_indexing/workers.rb
|
214
|
-
- lib/esse/async_indexing/workers/faktory.rb
|
215
|
-
- lib/esse/async_indexing/workers/shared_class_methods.rb
|
216
|
-
- lib/esse/async_indexing/workers/sidekiq.rb
|
217
204
|
- lib/esse/plugins/async_indexing.rb
|
218
205
|
homepage: https://github.com/marcosgz/esse-async_indexing
|
219
206
|
licenses:
|
@@ -233,9 +220,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
233
220
|
version: 2.7.0
|
234
221
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
235
222
|
requirements:
|
236
|
-
- - "
|
223
|
+
- - ">"
|
237
224
|
- !ruby/object:Gem::Version
|
238
|
-
version:
|
225
|
+
version: 1.3.1
|
239
226
|
requirements: []
|
240
227
|
rubygems_version: 3.2.33
|
241
228
|
signing_key:
|