esse-async_indexing 0.0.2 → 0.1.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/Gemfile +2 -1
- data/Gemfile.lock +18 -12
- data/README.md +50 -41
- data/lib/esse/async_indexing/actions/{batch_delete.rb → bulk_delete.rb} +2 -2
- data/lib/esse/async_indexing/actions/{batch_import.rb → bulk_import.rb} +1 -1
- data/lib/esse/async_indexing/actions/{batch_import_all.rb → bulk_import_all.rb} +1 -1
- data/lib/esse/async_indexing/actions/{batch_update.rb → bulk_update.rb} +6 -4
- data/lib/esse/async_indexing/actions/{update_lazy_document_attribute.rb → bulk_update_lazy_attribute.rb} +1 -1
- data/lib/esse/async_indexing/actions/delete_document.rb +1 -1
- data/lib/esse/async_indexing/actions/index_document.rb +1 -1
- data/lib/esse/async_indexing/actions/update_document.rb +1 -1
- data/lib/esse/async_indexing/actions/upsert_document.rb +1 -2
- data/lib/esse/async_indexing/actions.rb +5 -7
- data/lib/esse/async_indexing/active_record_callbacks/lazy_update_attribute.rb +8 -5
- data/lib/esse/async_indexing/cli/async_import.rb +27 -18
- data/lib/esse/async_indexing/cli/async_update_lazy_attributes.rb +78 -0
- data/lib/esse/async_indexing/cli.rb +23 -9
- data/lib/esse/async_indexing/config.rb +0 -3
- data/lib/esse/async_indexing/configuration.rb +54 -18
- data/lib/esse/async_indexing/jobs/bulk_update_lazy_attribute_job.rb +7 -0
- data/lib/esse/async_indexing/jobs/import_all_job.rb +1 -1
- data/lib/esse/async_indexing/jobs/import_ids_job.rb +42 -0
- data/lib/esse/async_indexing/{workers.rb → jobs.rb} +7 -24
- data/lib/esse/async_indexing/tasks.rb +82 -0
- data/lib/esse/async_indexing/version.rb +1 -1
- data/lib/esse/async_indexing.rb +15 -20
- data/lib/esse/plugins/async_indexing.rb +18 -59
- metadata +19 -32
- data/lib/esse/async_indexing/actions/bulk_update_lazy_document_attribute.rb +0 -20
- data/lib/esse/async_indexing/actions/import_batch_id.rb +0 -21
- data/lib/esse/async_indexing/adapters/adapter.rb +0 -29
- data/lib/esse/async_indexing/adapters/faktory.rb +0 -114
- data/lib/esse/async_indexing/adapters/sidekiq.rb +0 -94
- data/lib/esse/async_indexing/adapters.rb +0 -12
- data/lib/esse/async_indexing/configuration/base.rb +0 -65
- data/lib/esse/async_indexing/configuration/faktory.rb +0 -6
- data/lib/esse/async_indexing/configuration/sidekiq.rb +0 -12
- data/lib/esse/async_indexing/jobs/bulk_update_lazy_document_attribute_job.rb +0 -7
- data/lib/esse/async_indexing/jobs/import_batch_id_job.rb +0 -34
- data/lib/esse/async_indexing/jobs/update_lazy_document_attribute_job.rb +0 -7
- data/lib/esse/async_indexing/testing.rb +0 -79
- data/lib/esse/async_indexing/worker.rb +0 -85
- data/lib/esse/async_indexing/workers/faktory.rb +0 -28
- data/lib/esse/async_indexing/workers/shared_class_methods.rb +0 -26
- data/lib/esse/async_indexing/workers/sidekiq.rb +0 -28
@@ -13,20 +13,34 @@ Esse::CLI::Index.class_eval do
|
|
13
13
|
option :suffix, type: :string, default: nil, aliases: "-s", desc: "Suffix to append to index name"
|
14
14
|
option :context, type: :hash, default: {}, required: true, desc: "List of options to pass to the index class"
|
15
15
|
option :service, type: :string, default: nil, alias: "-s", desc: "Service to use for async import: sidekiq, faktory"
|
16
|
-
option :
|
17
|
-
option :
|
16
|
+
option :preload_lazy_attributes, type: :string, default: nil, desc: "Command separated list of lazy document attributes to preload using search API before the bulk import. Or pass `true` to preload all lazy attributes"
|
17
|
+
option :eager_load_lazy_attributes, type: :string, default: nil, desc: "Comma separated list of lazy document attributes to include to the bulk index request. Or pass `true` to include all lazy attributes"
|
18
|
+
option :update_lazy_attributes, type: :string, default: nil, desc: "Comma separated list of lazy document attributes to bulk update after the bulk index request Or pass `true` to include all lazy attributes"
|
19
|
+
option :enqueue_lazy_attributes, type: :boolean, default: nil, desc: "Enqueue the lazy document attributes job after the bulk import. (default: true))"
|
20
|
+
option :job_options, type: :hash, default: {}, desc: "List of options to pass to the background job. (Example: --job-options=queue:default)"
|
18
21
|
def async_import(*index_classes)
|
19
22
|
opts = Esse::HashUtils.deep_transform_keys(options.to_h, &:to_sym)
|
20
23
|
opts[:service] ||= Esse.config.async_indexing.services.first
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
end
|
26
|
-
if (val = opts[:lazy_update_document_attributes])
|
27
|
-
opts[:lazy_update_document_attributes] = (val == "true") ? true : val.split(",")
|
24
|
+
%i[preload_lazy_attributes eager_load_lazy_attributes update_lazy_attributes].each do |key|
|
25
|
+
if (val = opts.delete(key)) && val != "false"
|
26
|
+
opts[key] = (val == "true") ? true : val.split(",")
|
27
|
+
end
|
28
28
|
end
|
29
29
|
require "esse/async_indexing/cli/async_import"
|
30
30
|
Esse::AsyncIndexing::CLI::AsyncImport.new(indices: index_classes, **opts).run
|
31
31
|
end
|
32
|
+
|
33
|
+
desc "async_update_lazy_attributes INDEX_CLASS", "Async update lazy attributes for the given index"
|
34
|
+
option :repo, type: :string, default: nil, alias: "-r", desc: "Repository to use for import"
|
35
|
+
option :suffix, type: :string, default: nil, aliases: "-s", desc: "Suffix to append to index name"
|
36
|
+
option :context, type: :hash, default: {}, required: true, desc: "List of options to pass to the index class"
|
37
|
+
option :service, type: :string, default: nil, alias: "-s", desc: "Service to use for async import: sidekiq, faktory"
|
38
|
+
option :job_options, type: :hash, default: {}, desc: "List of options to pass to the background job. (Example: --job-options=queue:default)"
|
39
|
+
def async_update_lazy_attributes(index_class, *attributes)
|
40
|
+
opts = Esse::HashUtils.deep_transform_keys(options.to_h, &:to_sym)
|
41
|
+
opts[:service] ||= Esse.config.async_indexing.services.first
|
42
|
+
require "esse/async_indexing/cli/async_update_lazy_attributes"
|
43
|
+
|
44
|
+
Esse::AsyncIndexing::CLI::AsyncUpdateLazyAttributes.new(indices: [index_class], attributes: attributes, **opts).run
|
45
|
+
end
|
32
46
|
end
|
@@ -17,29 +17,65 @@ module Esse
|
|
17
17
|
@services ||= ConfigService.new
|
18
18
|
end
|
19
19
|
|
20
|
-
def
|
21
|
-
@
|
22
|
-
|
23
|
-
|
24
|
-
end
|
25
|
-
if block_given?
|
26
|
-
yield @faktory
|
27
|
-
else
|
28
|
-
@faktory
|
29
|
-
end
|
20
|
+
def reset!
|
21
|
+
@services = nil
|
22
|
+
@tasks = nil
|
23
|
+
bg_job_config.reset!
|
30
24
|
end
|
31
25
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
Configuration::Sidekiq.new
|
26
|
+
BackgroundJob::SERVICES.each_key do |service|
|
27
|
+
define_method(service) do |&block|
|
28
|
+
config_for(service, &block)
|
36
29
|
end
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
30
|
+
end
|
31
|
+
|
32
|
+
def config_for(service)
|
33
|
+
case service.to_sym
|
34
|
+
when :faktory
|
35
|
+
conf = bg_job_config.faktory
|
36
|
+
unless services.faktory?
|
37
|
+
default_jobs.each { |job_class| conf.jobs[job_class] ||= {} }
|
38
|
+
services << :faktory
|
39
|
+
end
|
40
|
+
yield conf if block_given?
|
41
|
+
conf
|
42
|
+
when :sidekiq
|
43
|
+
conf = bg_job_config.sidekiq
|
44
|
+
unless services.sidekiq?
|
45
|
+
default_jobs.each { |job_class| conf.jobs[job_class] ||= {} }
|
46
|
+
services << :sidekiq
|
47
|
+
end
|
48
|
+
yield conf if block_given?
|
49
|
+
conf
|
50
|
+
else raise ArgumentError, "Unknown service: #{service}"
|
41
51
|
end
|
42
52
|
end
|
53
|
+
|
54
|
+
def tasks
|
55
|
+
@tasks ||= Esse::AsyncIndexing::Tasks.new
|
56
|
+
end
|
57
|
+
|
58
|
+
# DSL to define custom job enqueueing
|
59
|
+
#
|
60
|
+
# task(:import) do |service:, repo:, operation:, ids:, **kwargs|
|
61
|
+
# MyCustomJob.perform_later(repo.index.name, ids, **kwargs)
|
62
|
+
# end
|
63
|
+
# task(:index, :update, :delete) do |service:, repo:, operation:, id, **kwargs|
|
64
|
+
# MyCustomJob.perform_later(repo.index.name, [id], **kwargs)
|
65
|
+
# end
|
66
|
+
def task(*operations, &block)
|
67
|
+
tasks.define(*operations, &block)
|
68
|
+
end
|
69
|
+
|
70
|
+
private
|
71
|
+
|
72
|
+
def bg_job_config
|
73
|
+
BackgroundJob.config
|
74
|
+
end
|
75
|
+
|
76
|
+
def default_jobs
|
77
|
+
Esse::AsyncIndexing::Jobs::DEFAULT.values
|
78
|
+
end
|
43
79
|
end
|
44
80
|
end
|
45
81
|
end
|
@@ -0,0 +1,7 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob
|
4
|
+
def perform(index_class_name, repo_name, attribute_name, ids, options = {})
|
5
|
+
Esse::AsyncIndexing::Actions::BulkUpdateLazyAttribute.call(index_class_name, repo_name, attribute_name, ids, options)
|
6
|
+
end
|
7
|
+
end
|
@@ -2,6 +2,6 @@
|
|
2
2
|
|
3
3
|
class Esse::AsyncIndexing::Jobs::ImportAllJob
|
4
4
|
def perform(index_class_name, repo_name, options = {})
|
5
|
-
Esse::AsyncIndexing::Actions::
|
5
|
+
Esse::AsyncIndexing::Actions::BulkImportAll.call(index_class_name, repo_name, options)
|
6
6
|
end
|
7
7
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Esse::AsyncIndexing::Jobs::ImportIdsJob
|
4
|
+
def perform(index_class_name, repo_name, ids, options = {})
|
5
|
+
# This is specific to the AsyncIndexing plugin, can't pass to Esse import method
|
6
|
+
enqueue_lazy = options.delete(:enqueue_lazy_attributes) if options.key?(:enqueue_lazy_attributes)
|
7
|
+
enqueue_lazy = options.delete("enqueue_lazy_attributes") if options.key?("enqueue_lazy_attributes")
|
8
|
+
enqueue_lazy = true if enqueue_lazy.nil?
|
9
|
+
total = Esse::AsyncIndexing::Actions::BulkImport.call(index_class_name, repo_name, ids, options)
|
10
|
+
options = Esse::HashUtils.deep_transform_keys(options, &:to_s)
|
11
|
+
|
12
|
+
return total if total.zero?
|
13
|
+
return total unless enqueue_lazy
|
14
|
+
return total if lazy_already_imported?(options)
|
15
|
+
return total unless self.class.respond_to?(:background_job_service)
|
16
|
+
|
17
|
+
index_class, repo_class = Esse::AsyncIndexing::Actions::CoerceIndexRepository.call(index_class_name, repo_name)
|
18
|
+
return total unless Esse::AsyncIndexing.plugin_installed?(index_class)
|
19
|
+
|
20
|
+
repo_class.lazy_document_attributes.each_key do |attr_name|
|
21
|
+
repo_class.async_indexing_job_for(:update_lazy_attribute).call(
|
22
|
+
**options,
|
23
|
+
service: self.class.background_job_service,
|
24
|
+
repo: repo_class,
|
25
|
+
operation: :update_lazy_attribute,
|
26
|
+
attribute: attr_name,
|
27
|
+
ids: ids
|
28
|
+
)
|
29
|
+
end
|
30
|
+
total
|
31
|
+
end
|
32
|
+
|
33
|
+
protected
|
34
|
+
|
35
|
+
# The `import` action already eager or lazy load the document attributes when some of these options are set.
|
36
|
+
def lazy_already_imported?(options)
|
37
|
+
preload = options.delete("preload_lazy_attributes") || false
|
38
|
+
eager = options.delete("eager_load_lazy_attributes") || false
|
39
|
+
lazy = options.delete("update_lazy_attributes") || false
|
40
|
+
preload || eager || lazy
|
41
|
+
end
|
42
|
+
end
|
@@ -1,20 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require_relative "worker"
|
4
|
-
|
5
3
|
module Esse
|
6
4
|
module AsyncIndexing
|
7
|
-
module
|
5
|
+
module Jobs
|
8
6
|
DEFAULT = {
|
9
|
-
"esse/async_indexing/jobs/
|
7
|
+
"esse/async_indexing/jobs/bulk_update_lazy_attribute_job" => "Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob",
|
10
8
|
"esse/async_indexing/jobs/document_delete_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob",
|
11
9
|
"esse/async_indexing/jobs/document_index_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob",
|
12
10
|
"esse/async_indexing/jobs/document_update_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob",
|
13
11
|
"esse/async_indexing/jobs/document_upsert_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentUpsertByIdJob",
|
14
12
|
"esse/async_indexing/jobs/import_all_job" => "Esse::AsyncIndexing::Jobs::ImportAllJob",
|
15
|
-
"esse/async_indexing/jobs/
|
16
|
-
|
17
|
-
}
|
13
|
+
"esse/async_indexing/jobs/import_ids_job" => "Esse::AsyncIndexing::Jobs::ImportIdsJob"
|
14
|
+
}.freeze
|
18
15
|
|
19
16
|
# The backend service may live in a different application, so they are not required by default.
|
20
17
|
# That's why we have a separate structure to let enqueue jobs even without having the explicit worker class loaded.
|
@@ -22,27 +19,13 @@ module Esse
|
|
22
19
|
def self.install!(service, **options)
|
23
20
|
return if @installed_services&.include?(service.to_sym)
|
24
21
|
|
25
|
-
DEFAULT.each do |job,
|
22
|
+
DEFAULT.each do |job, const_name|
|
26
23
|
Kernel.require(job)
|
27
|
-
|
28
|
-
|
24
|
+
klass = Esse::AsyncIndexing::Jobs.const_get(const_name.split("::").last)
|
25
|
+
klass.extend BackgroundJob.mixin(service, **options)
|
29
26
|
end
|
30
27
|
@installed_services = Array(@installed_services) << service.to_sym
|
31
28
|
end
|
32
|
-
|
33
|
-
def self.for(service, **options)
|
34
|
-
require_relative "workers/#{service}"
|
35
|
-
service = service.to_sym
|
36
|
-
worker_options = options.merge(service: service)
|
37
|
-
module_name = service.to_s.split(/_/i).collect! { |w| w.capitalize }.join
|
38
|
-
mod = Esse::AsyncIndexing::Workers.const_get(module_name)
|
39
|
-
mod.module_eval do
|
40
|
-
define_method(:bg_worker_options) do
|
41
|
-
worker_options
|
42
|
-
end
|
43
|
-
end
|
44
|
-
mod
|
45
|
-
end
|
46
29
|
end
|
47
30
|
end
|
48
31
|
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Esse
|
4
|
+
module AsyncIndexing
|
5
|
+
class Tasks
|
6
|
+
DEFAULT = {
|
7
|
+
import: ->(service:, repo:, operation:, ids:, job_options: {}, **kwargs) {
|
8
|
+
unless (ids = Esse::ArrayUtils.wrap(ids)).empty?
|
9
|
+
BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::ImportIdsJob", **job_options)
|
10
|
+
.with_args(repo.index.name, repo.repo_name, ids, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
11
|
+
.push
|
12
|
+
end
|
13
|
+
},
|
14
|
+
index: ->(service:, repo:, operation:, id:, job_options: {}, **kwargs) {
|
15
|
+
if id
|
16
|
+
BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob", **job_options)
|
17
|
+
.with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
18
|
+
.push
|
19
|
+
end
|
20
|
+
},
|
21
|
+
update: ->(service:, repo:, operation:, id:, job_options: {}, **kwargs) {
|
22
|
+
if id
|
23
|
+
BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob", **job_options)
|
24
|
+
.with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
25
|
+
.push
|
26
|
+
end
|
27
|
+
},
|
28
|
+
delete: ->(service:, repo:, operation:, id:, job_options: {}, **kwargs) {
|
29
|
+
if id
|
30
|
+
BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob", **job_options)
|
31
|
+
.with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
32
|
+
.push
|
33
|
+
end
|
34
|
+
},
|
35
|
+
update_lazy_attribute: ->(service:, repo:, operation:, attribute:, ids:, job_options: {}, **kwargs) {
|
36
|
+
unless (ids = Esse::ArrayUtils.wrap(ids)).empty?
|
37
|
+
BackgroundJob.job(service, "Esse::AsyncIndexing::Jobs::BulkUpdateLazyAttributeJob", **job_options)
|
38
|
+
.with_args(repo.index.name, repo.repo_name, attribute.to_s, ids, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
39
|
+
.push
|
40
|
+
end
|
41
|
+
}
|
42
|
+
}.freeze
|
43
|
+
|
44
|
+
def initialize
|
45
|
+
@tasks = {}.freeze
|
46
|
+
end
|
47
|
+
|
48
|
+
def user_defined?(name)
|
49
|
+
@tasks.key?(name.to_sym)
|
50
|
+
end
|
51
|
+
|
52
|
+
def define(*names, &block)
|
53
|
+
names = DEFAULT.keys if names.empty?
|
54
|
+
validate!(names, block)
|
55
|
+
new_tasks = names.each_with_object({}) { |name, h| h[name.to_sym] = block }
|
56
|
+
@tasks = @tasks.dup.merge(new_tasks)
|
57
|
+
ensure
|
58
|
+
@tasks.freeze
|
59
|
+
end
|
60
|
+
|
61
|
+
def fetch(name)
|
62
|
+
id = name.to_sym
|
63
|
+
@tasks[id] || DEFAULT[id] || raise(ArgumentError, "Unknown task: #{name}")
|
64
|
+
end
|
65
|
+
alias_method :[], :fetch
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
def validate!(names, block)
|
70
|
+
unless block.is_a?(Proc)
|
71
|
+
raise ArgumentError, "The block of task must be a callable object"
|
72
|
+
end
|
73
|
+
|
74
|
+
names.each do |name|
|
75
|
+
unless DEFAULT.key?(name)
|
76
|
+
raise ArgumentError, "Unrecognized task: #{name}. Valid tasks are: #{DEFAULT.keys}"
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
data/lib/esse/async_indexing.rb
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "esse"
|
4
|
-
require "esse-redis_storage"
|
5
4
|
require "forwardable"
|
6
5
|
require "securerandom"
|
7
6
|
require "time"
|
8
7
|
require "multi_json"
|
8
|
+
require "background_job"
|
9
9
|
|
10
10
|
module Esse
|
11
11
|
module AsyncIndexing
|
@@ -14,32 +14,24 @@ module Esse
|
|
14
14
|
|
15
15
|
module Jobs
|
16
16
|
end
|
17
|
-
|
18
|
-
module Workers
|
19
|
-
end
|
20
17
|
end
|
21
18
|
end
|
22
19
|
|
23
20
|
require_relative "async_indexing/version"
|
24
21
|
require_relative "async_indexing/actions"
|
25
|
-
require_relative "async_indexing/adapters"
|
26
22
|
require_relative "async_indexing/config"
|
23
|
+
require_relative "async_indexing/tasks"
|
27
24
|
require_relative "async_indexing/errors"
|
28
|
-
require_relative "async_indexing/
|
25
|
+
require_relative "async_indexing/jobs"
|
29
26
|
require_relative "plugins/async_indexing"
|
30
27
|
|
31
28
|
module Esse::AsyncIndexing
|
32
|
-
SERVICES = {
|
33
|
-
sidekiq: Adapters::Sidekiq,
|
34
|
-
faktory: Adapters::Faktory
|
35
|
-
}
|
36
|
-
|
37
29
|
# @param worker_class [String] The worker class name
|
38
30
|
# @param options [Hash] Options that will be passed along to the worker instance
|
39
31
|
# @return [Esse::AsyncIndexing::Worker] An instance of worker
|
40
|
-
def self.worker(worker_class, service
|
32
|
+
def self.worker(worker_class, service:, **options)
|
41
33
|
serv_name = service_name(service)
|
42
|
-
|
34
|
+
BackgroundJob.job(serv_name, worker_class, **options)
|
43
35
|
end
|
44
36
|
|
45
37
|
def self.service_name(identifier = nil)
|
@@ -47,26 +39,29 @@ module Esse::AsyncIndexing
|
|
47
39
|
if identifier.nil?
|
48
40
|
raise ArgumentError, "There are no async indexing services configured. Please configure at least one service or pass the service name as an argument."
|
49
41
|
end
|
50
|
-
|
51
|
-
|
52
|
-
raise ArgumentError, "Invalid service: #{identifier.inspect}, valid services are: #{SERVICES.keys.join(", ")}"
|
42
|
+
unless (services = BackgroundJob::SERVICES).key?(identifier.to_sym)
|
43
|
+
raise ArgumentError, "Invalid service: #{identifier.inspect}, valid services are: #{services.keys.join(", ")}"
|
53
44
|
end
|
54
45
|
|
55
46
|
identifier.to_sym
|
56
47
|
end
|
57
48
|
|
58
|
-
def self.jid
|
59
|
-
SecureRandom.hex(12)
|
60
|
-
end
|
61
|
-
|
62
49
|
def self.async_indexing_repo?(repo)
|
63
50
|
return false unless repo.is_a?(Class) && repo < Esse::Repository
|
64
51
|
|
65
52
|
repo.respond_to?(:implement_batch_ids?) && repo.implement_batch_ids?
|
66
53
|
end
|
54
|
+
|
55
|
+
def self.plugin_installed?(index)
|
56
|
+
index = index.index if index.is_a?(Class) && index < Esse::Repository
|
57
|
+
return false unless index.is_a?(Class) && index < Esse::Index
|
58
|
+
|
59
|
+
index.plugins.include?(Esse::Plugins::AsyncIndexing)
|
60
|
+
end
|
67
61
|
end
|
68
62
|
|
69
63
|
Esse::Config.__send__ :include, Esse::AsyncIndexing::Config
|
64
|
+
|
70
65
|
if defined?(Esse::CLI)
|
71
66
|
require_relative "async_indexing/cli"
|
72
67
|
end
|
@@ -4,48 +4,23 @@ module Esse
|
|
4
4
|
module Plugins
|
5
5
|
module AsyncIndexing
|
6
6
|
module RepositoryClassMethods
|
7
|
-
DEFAULT_ASYNC_INDEXING_JOBS = {
|
8
|
-
import: ->(service:, repo:, operation:, ids:, **kwargs) {
|
9
|
-
unless (ids = Esse::ArrayUtils.wrap(ids)).empty?
|
10
|
-
batch_id = Esse::RedisStorage::Queue.for(repo: repo).enqueue(values: ids)
|
11
|
-
Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::ImportBatchIdJob", service: service)
|
12
|
-
.with_args(repo.index.name, repo.repo_name, batch_id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
13
|
-
.push
|
14
|
-
end
|
15
|
-
},
|
16
|
-
index: ->(service:, repo:, operation:, id:, **kwargs) {
|
17
|
-
if id
|
18
|
-
Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob", service: service)
|
19
|
-
.with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
20
|
-
.push
|
21
|
-
end
|
22
|
-
},
|
23
|
-
update: ->(service:, repo:, operation:, id:, **kwargs) {
|
24
|
-
if id
|
25
|
-
Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob", service: service)
|
26
|
-
.with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
27
|
-
.push
|
28
|
-
end
|
29
|
-
},
|
30
|
-
delete: ->(service:, repo:, operation:, id:, **kwargs) {
|
31
|
-
if id
|
32
|
-
Esse::AsyncIndexing.worker("Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob", service: service)
|
33
|
-
.with_args(repo.index.name, repo.repo_name, id, Esse::HashUtils.deep_transform_keys(kwargs, &:to_s))
|
34
|
-
.push
|
35
|
-
end
|
36
|
-
}
|
37
|
-
}.freeze
|
38
|
-
|
39
7
|
# This method is used to retrieve only the ids of the documents in the collection.
|
40
8
|
# It's used to asynchronously index the documents.
|
41
9
|
# The #each_batch_ids method is optional and should be implemented by the collection class.
|
42
10
|
#
|
11
|
+
# @yield [batch] Yields each batch of ids
|
43
12
|
# @return [Enumerator] The enumerator
|
44
|
-
def batch_ids(*args, **kwargs
|
13
|
+
def batch_ids(*args, **kwargs)
|
45
14
|
if implement_batch_ids?
|
46
|
-
|
15
|
+
if block_given?
|
47
16
|
@collection_proc.new(*args, **kwargs).each_batch_ids do |batch|
|
48
|
-
|
17
|
+
yield(batch)
|
18
|
+
end
|
19
|
+
else
|
20
|
+
Enumerator.new do |yielder|
|
21
|
+
@collection_proc.new(*args, **kwargs).each_batch_ids do |batch|
|
22
|
+
yielder.yield(batch)
|
23
|
+
end
|
49
24
|
end
|
50
25
|
end
|
51
26
|
else
|
@@ -70,35 +45,19 @@ module Esse
|
|
70
45
|
# MyCustomJob.perform_later(repo.index.name, [id], **kwargs)
|
71
46
|
# end
|
72
47
|
def async_indexing_job(*operations, &block)
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
@async_indexing_jobs = async_indexing_jobs.dup.merge(hash)
|
77
|
-
ensure
|
78
|
-
@async_indexing_jobs.freeze
|
48
|
+
definer = @async_indexing_tasks || Esse::AsyncIndexing::Tasks.new
|
49
|
+
definer.define(*operations, &block)
|
50
|
+
@async_indexing_tasks = definer
|
79
51
|
end
|
80
52
|
|
81
|
-
def
|
82
|
-
|
83
|
-
end
|
53
|
+
def async_indexing_job?(operation)
|
54
|
+
return false unless @async_indexing_tasks
|
84
55
|
|
85
|
-
|
86
|
-
async_indexing_jobs[operation] || DEFAULT_ASYNC_INDEXING_JOBS[operation] || raise(ArgumentError, "The #{operation} operation is not implemented")
|
56
|
+
@async_indexing_tasks.user_defined?(operation)
|
87
57
|
end
|
88
58
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
def self.call(operations, block)
|
93
|
-
unless block.is_a?(Proc)
|
94
|
-
raise ArgumentError, "The block of async_indexing_job must be a callable object"
|
95
|
-
end
|
96
|
-
|
97
|
-
operations.each do |operation|
|
98
|
-
next if OPERATIONS.include?(operation)
|
99
|
-
raise ArgumentError, format("Unrecognized operation: %<operation>p. Valid operations are: %<valid>p", operation: operation, valid: OPERATIONS)
|
100
|
-
end
|
101
|
-
end
|
59
|
+
def async_indexing_job_for(operation)
|
60
|
+
(@async_indexing_tasks || Esse.config.async_indexing.tasks).fetch(operation)
|
102
61
|
end
|
103
62
|
end
|
104
63
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: esse-async_indexing
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.1.0.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Marcos G. Zimmermann
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-08-
|
11
|
+
date: 2024-08-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: esse
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.
|
19
|
+
version: 0.4.0.rc1
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0.
|
26
|
+
version: 0.4.0.rc1
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: multi_json
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -39,19 +39,19 @@ dependencies:
|
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: 0.0.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: background_job
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.
|
47
|
+
version: 0.0.0
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.
|
54
|
+
version: 0.0.0
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: pry
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -169,17 +169,15 @@ files:
|
|
169
169
|
- lib/esse-async_indexing.rb
|
170
170
|
- lib/esse/async_indexing.rb
|
171
171
|
- lib/esse/async_indexing/actions.rb
|
172
|
-
- lib/esse/async_indexing/actions/
|
173
|
-
- lib/esse/async_indexing/actions/
|
174
|
-
- lib/esse/async_indexing/actions/
|
175
|
-
- lib/esse/async_indexing/actions/
|
176
|
-
- lib/esse/async_indexing/actions/
|
172
|
+
- lib/esse/async_indexing/actions/bulk_delete.rb
|
173
|
+
- lib/esse/async_indexing/actions/bulk_import.rb
|
174
|
+
- lib/esse/async_indexing/actions/bulk_import_all.rb
|
175
|
+
- lib/esse/async_indexing/actions/bulk_update.rb
|
176
|
+
- lib/esse/async_indexing/actions/bulk_update_lazy_attribute.rb
|
177
177
|
- lib/esse/async_indexing/actions/coerce_index_repository.rb
|
178
178
|
- lib/esse/async_indexing/actions/delete_document.rb
|
179
|
-
- lib/esse/async_indexing/actions/import_batch_id.rb
|
180
179
|
- lib/esse/async_indexing/actions/index_document.rb
|
181
180
|
- lib/esse/async_indexing/actions/update_document.rb
|
182
|
-
- lib/esse/async_indexing/actions/update_lazy_document_attribute.rb
|
183
181
|
- lib/esse/async_indexing/actions/upsert_document.rb
|
184
182
|
- lib/esse/async_indexing/active_record.rb
|
185
183
|
- lib/esse/async_indexing/active_record_callbacks/callback.rb
|
@@ -187,33 +185,22 @@ files:
|
|
187
185
|
- lib/esse/async_indexing/active_record_callbacks/on_create.rb
|
188
186
|
- lib/esse/async_indexing/active_record_callbacks/on_destroy.rb
|
189
187
|
- lib/esse/async_indexing/active_record_callbacks/on_update.rb
|
190
|
-
- lib/esse/async_indexing/adapters.rb
|
191
|
-
- lib/esse/async_indexing/adapters/adapter.rb
|
192
|
-
- lib/esse/async_indexing/adapters/faktory.rb
|
193
|
-
- lib/esse/async_indexing/adapters/sidekiq.rb
|
194
188
|
- lib/esse/async_indexing/cli.rb
|
195
189
|
- lib/esse/async_indexing/cli/async_import.rb
|
190
|
+
- lib/esse/async_indexing/cli/async_update_lazy_attributes.rb
|
196
191
|
- lib/esse/async_indexing/config.rb
|
197
192
|
- lib/esse/async_indexing/configuration.rb
|
198
|
-
- lib/esse/async_indexing/configuration/base.rb
|
199
|
-
- lib/esse/async_indexing/configuration/faktory.rb
|
200
|
-
- lib/esse/async_indexing/configuration/sidekiq.rb
|
201
193
|
- lib/esse/async_indexing/errors.rb
|
202
|
-
- lib/esse/async_indexing/jobs
|
194
|
+
- lib/esse/async_indexing/jobs.rb
|
195
|
+
- lib/esse/async_indexing/jobs/bulk_update_lazy_attribute_job.rb
|
203
196
|
- lib/esse/async_indexing/jobs/document_delete_by_id_job.rb
|
204
197
|
- lib/esse/async_indexing/jobs/document_index_by_id_job.rb
|
205
198
|
- lib/esse/async_indexing/jobs/document_update_by_id_job.rb
|
206
199
|
- lib/esse/async_indexing/jobs/document_upsert_by_id_job.rb
|
207
200
|
- lib/esse/async_indexing/jobs/import_all_job.rb
|
208
|
-
- lib/esse/async_indexing/jobs/
|
209
|
-
- lib/esse/async_indexing/
|
210
|
-
- lib/esse/async_indexing/testing.rb
|
201
|
+
- lib/esse/async_indexing/jobs/import_ids_job.rb
|
202
|
+
- lib/esse/async_indexing/tasks.rb
|
211
203
|
- lib/esse/async_indexing/version.rb
|
212
|
-
- lib/esse/async_indexing/worker.rb
|
213
|
-
- lib/esse/async_indexing/workers.rb
|
214
|
-
- lib/esse/async_indexing/workers/faktory.rb
|
215
|
-
- lib/esse/async_indexing/workers/shared_class_methods.rb
|
216
|
-
- lib/esse/async_indexing/workers/sidekiq.rb
|
217
204
|
- lib/esse/plugins/async_indexing.rb
|
218
205
|
homepage: https://github.com/marcosgz/esse-async_indexing
|
219
206
|
licenses:
|
@@ -233,9 +220,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
233
220
|
version: 2.7.0
|
234
221
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
235
222
|
requirements:
|
236
|
-
- - "
|
223
|
+
- - ">"
|
237
224
|
- !ruby/object:Gem::Version
|
238
|
-
version:
|
225
|
+
version: 1.3.1
|
239
226
|
requirements: []
|
240
227
|
rubygems_version: 3.2.33
|
241
228
|
signing_key:
|