connectors_service 8.6.0.4.pre.20221116T024501Z → 8.6.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/config/connectors.yml +6 -6
- data/lib/app/app.rb +0 -4
- data/lib/app/dispatcher.rb +17 -42
- data/lib/app/preflight_check.rb +0 -11
- data/lib/connectors/base/connector.rb +14 -43
- data/lib/connectors/example/connector.rb +0 -6
- data/lib/connectors/gitlab/connector.rb +1 -6
- data/lib/connectors/mongodb/connector.rb +43 -47
- data/lib/connectors/sync_status.rb +1 -6
- data/lib/core/configuration.rb +1 -3
- data/lib/core/connector_settings.rb +16 -52
- data/lib/core/elastic_connector_actions.rb +59 -320
- data/lib/core/output_sink/base_sink.rb +33 -0
- data/lib/core/output_sink/combined_sink.rb +38 -0
- data/lib/core/output_sink/console_sink.rb +51 -0
- data/lib/core/output_sink/es_sink.rb +74 -0
- data/lib/core/{ingestion.rb → output_sink.rb} +5 -1
- data/lib/core/scheduler.rb +10 -40
- data/lib/core/single_scheduler.rb +1 -1
- data/lib/core/sync_job_runner.rb +16 -72
- data/lib/core.rb +0 -4
- data/lib/utility/constants.rb +0 -2
- data/lib/utility/errors.rb +12 -0
- data/lib/utility/logger.rb +1 -1
- data/lib/utility.rb +4 -11
- metadata +9 -27
- data/lib/connectors/base/advanced_snippet_against_schema_validator.rb +0 -173
- data/lib/connectors/base/advanced_snippet_validator.rb +0 -34
- data/lib/connectors/base/simple_rules_parser.rb +0 -42
- data/lib/connectors/example/example_advanced_snippet_validator.rb +0 -35
- data/lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb +0 -35
- data/lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb +0 -22
- data/lib/connectors/mongodb/mongo_advanced_snippet_schema.rb +0 -292
- data/lib/connectors/mongodb/mongo_rules_parser.rb +0 -81
- data/lib/connectors/tolerable_error_helper.rb +0 -43
- data/lib/core/connector_job.rb +0 -210
- data/lib/core/filtering/post_process_engine.rb +0 -39
- data/lib/core/filtering/post_process_result.rb +0 -27
- data/lib/core/filtering/simple_rule.rb +0 -141
- data/lib/core/filtering/validation_job_runner.rb +0 -53
- data/lib/core/filtering/validation_status.rb +0 -17
- data/lib/core/filtering.rb +0 -17
- data/lib/core/ingestion/es_sink.rb +0 -118
- data/lib/core/jobs/consumer.rb +0 -114
- data/lib/core/jobs/producer.rb +0 -26
- data/lib/utility/bulk_queue.rb +0 -85
- data/lib/utility/error_monitor.rb +0 -108
- data/lib/utility/filtering.rb +0 -22
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 43cb60284b3e1c74a6f34612c0f7509cb4be0375d5871c6b59d7338495c88236
|
4
|
+
data.tar.gz: 3f176c8770fc9b450a244b5f8cc7a5857668a452c17e19a07ddb412dcb8d2452
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 59746d5ae600b0a77aaede033f5b371cd922e3a4e95f61afda146f5b6e0962026c02c987fc43c203c5274381539f5c76e5ef895fce593ef1883fa6cf620a1702
|
7
|
+
data.tar.gz: ee799a965b0f5b028c2a41003069d3b7b1e49cabe38453629af8a382ed7ac92b76fe835986c7e5157c1a06b1c2ebae84c20606319bbe7b0aad9a771b12a3f977
|
data/config/connectors.yml
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
# general metadata
|
2
|
-
version: 8.6.0.4-
|
2
|
+
version: 8.6.0.4-20221104T200420Z
|
3
3
|
repository: git@github.com:elastic/ent-search-connectors.git
|
4
|
-
revision:
|
4
|
+
revision: 2051b3907639a1fe2ae68efdc33c06cf12d38383
|
5
5
|
elasticsearch:
|
6
6
|
hosts: http://localhost:9200
|
7
|
-
api_key:
|
7
|
+
api_key: OW1FalJJUUI1clBtUVh5RVo1QmU6QVp5LV9pU3RRUXFYb2VVYnlCRWNZdw==
|
8
8
|
retry_on_failure: 3
|
9
9
|
request_timeout: 120
|
10
10
|
disable_warnings: true
|
@@ -14,11 +14,11 @@ thread_pool:
|
|
14
14
|
min_threads: 0
|
15
15
|
max_threads: 5
|
16
16
|
max_queue: 100
|
17
|
-
log_level:
|
17
|
+
log_level: info
|
18
18
|
ecs_logging: false
|
19
19
|
poll_interval: 3
|
20
20
|
termination_timeout: 60
|
21
21
|
heartbeat_interval: 1800
|
22
22
|
native_mode: false
|
23
|
-
connector_id:
|
24
|
-
service_type:
|
23
|
+
connector_id: 9WEjRIQB5rPmQXyEWJB2
|
24
|
+
service_type: example
|
data/lib/app/app.rb
CHANGED
@@ -17,10 +17,6 @@ require 'utility/logger'
|
|
17
17
|
module App
|
18
18
|
Utility::Environment.set_execution_environment(App::Config) do
|
19
19
|
App::PreflightCheck.run!
|
20
|
-
|
21
|
-
# set exit hook
|
22
|
-
Kernel.at_exit { App::Dispatcher.shutdown! }
|
23
|
-
|
24
20
|
App::Dispatcher.start!
|
25
21
|
rescue App::PreflightCheck::CheckFailure => e
|
26
22
|
Utility::Logger.error("Preflight check failed: #{e.message}")
|
data/lib/app/dispatcher.rb
CHANGED
@@ -28,10 +28,6 @@ module App
|
|
28
28
|
def start!
|
29
29
|
running!
|
30
30
|
Utility::Logger.info("Starting connector service in #{App::Config.native_mode ? 'native' : 'non-native'} mode...")
|
31
|
-
|
32
|
-
# start sync jobs consumer
|
33
|
-
start_consumer!
|
34
|
-
|
35
31
|
start_polling_jobs!
|
36
32
|
end
|
37
33
|
|
@@ -41,8 +37,6 @@ module App
|
|
41
37
|
scheduler.shutdown
|
42
38
|
pool.shutdown
|
43
39
|
pool.wait_for_termination(TERMINATION_TIMEOUT)
|
44
|
-
|
45
|
-
stop_consumer!
|
46
40
|
end
|
47
41
|
|
48
42
|
private
|
@@ -74,16 +68,11 @@ module App
|
|
74
68
|
scheduler.when_triggered do |connector_settings, task|
|
75
69
|
case task
|
76
70
|
when :sync
|
77
|
-
|
78
|
-
Core::ElasticConnectorActions.update_connector_sync_now(connector_settings.id, false)
|
79
|
-
|
80
|
-
Core::Jobs::Producer.enqueue_job(job_type: :sync, connector_settings: connector_settings)
|
71
|
+
start_sync_task(connector_settings)
|
81
72
|
when :heartbeat
|
82
73
|
start_heartbeat_task(connector_settings)
|
83
74
|
when :configuration
|
84
75
|
start_configuration_task(connector_settings)
|
85
|
-
when :filter_validation
|
86
|
-
start_filter_validation_task(connector_settings)
|
87
76
|
else
|
88
77
|
Utility::Logger.error("Unknown task type: #{task}. Skipping...")
|
89
78
|
end
|
@@ -92,6 +81,22 @@ module App
|
|
92
81
|
Utility::ExceptionTracking.log_exception(e, 'The connector service failed due to unexpected error.')
|
93
82
|
end
|
94
83
|
|
84
|
+
def start_sync_task(connector_settings)
|
85
|
+
start_heartbeat_task(connector_settings)
|
86
|
+
pool.post do
|
87
|
+
Utility::Logger.info("Initiating a sync job for #{connector_settings.formatted}...")
|
88
|
+
Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
|
89
|
+
job_runner = Core::SyncJobRunner.new(connector_settings)
|
90
|
+
job_runner.execute
|
91
|
+
rescue Core::JobAlreadyRunningError
|
92
|
+
Utility::Logger.info("Sync job for #{connector_settings.formatted} is already running, skipping.")
|
93
|
+
rescue Core::ConnectorVersionChangedError => e
|
94
|
+
Utility::Logger.info("Could not start the job because #{connector_settings.formatted} has been updated externally. Message: #{e.message}")
|
95
|
+
rescue StandardError => e
|
96
|
+
Utility::ExceptionTracking.log_exception(e, "Sync job for #{connector_settings.formatted} failed due to unexpected error.")
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
95
100
|
def start_heartbeat_task(connector_settings)
|
96
101
|
pool.post do
|
97
102
|
Utility::Logger.info("Sending heartbeat for #{connector_settings.formatted}...")
|
@@ -115,36 +120,6 @@ module App
|
|
115
120
|
Utility::ExceptionTracking.log_exception(e, "Configuration task for #{connector_settings.formatted} failed due to unexpected error.")
|
116
121
|
end
|
117
122
|
end
|
118
|
-
|
119
|
-
def start_filter_validation_task(connector_settings)
|
120
|
-
pool.post do
|
121
|
-
Utility::Logger.info("Validating filters for #{connector_settings.formatted}...")
|
122
|
-
validation_job_runner = Core::Filtering::ValidationJobRunner.new(connector_settings)
|
123
|
-
validation_job_runner.execute
|
124
|
-
rescue StandardError => e
|
125
|
-
Utility::ExceptionTracking.log_exception(e, "Filter validation task for #{connector_settings.formatted} failed due to unexpected error.")
|
126
|
-
end
|
127
|
-
end
|
128
|
-
|
129
|
-
def start_consumer!
|
130
|
-
@consumer = Core::Jobs::Consumer.new(
|
131
|
-
poll_interval: POLL_INTERVAL,
|
132
|
-
termination_timeout: TERMINATION_TIMEOUT,
|
133
|
-
min_threads: MIN_THREADS,
|
134
|
-
max_threads: MAX_THREADS,
|
135
|
-
max_queue: MAX_QUEUE,
|
136
|
-
scheduler: scheduler
|
137
|
-
)
|
138
|
-
|
139
|
-
@consumer.subscribe!(index_name: Utility::Constants::JOB_INDEX)
|
140
|
-
end
|
141
|
-
|
142
|
-
def stop_consumer!
|
143
|
-
return if @consumer.nil?
|
144
|
-
return unless @consumer.running?
|
145
|
-
|
146
|
-
@consumer.shutdown!
|
147
|
-
end
|
148
123
|
end
|
149
124
|
end
|
150
125
|
end
|
data/lib/app/preflight_check.rb
CHANGED
@@ -23,7 +23,6 @@ module App
|
|
23
23
|
check_es_connection!
|
24
24
|
check_es_version!
|
25
25
|
check_system_indices!
|
26
|
-
check_single_connector!
|
27
26
|
end
|
28
27
|
|
29
28
|
private
|
@@ -60,16 +59,6 @@ module App
|
|
60
59
|
)
|
61
60
|
end
|
62
61
|
|
63
|
-
#-------------------------------------------------------------------------------------------------
|
64
|
-
# Ensures the connector is supported when running in non-native mode
|
65
|
-
def check_single_connector!
|
66
|
-
if App::Config.native_mode
|
67
|
-
Utility::Logger.info('Skip single connector check for native mode.')
|
68
|
-
elsif !Connectors::REGISTRY.registered?(App::Config.service_type)
|
69
|
-
fail_check!("The service type #{App::Config.service_type} is not supported. Terminating...")
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
62
|
def check_es_connection_with_retries!(retry_interval:, retry_timeout:)
|
74
63
|
started_at = Time.now
|
75
64
|
|
@@ -6,15 +6,11 @@
|
|
6
6
|
|
7
7
|
# frozen_string_literal: true
|
8
8
|
|
9
|
-
require 'active_support/core_ext/hash/indifferent_access'
|
10
|
-
require 'app/config'
|
11
9
|
require 'bson'
|
12
|
-
require '
|
13
|
-
require 'core/ingestion'
|
14
|
-
require 'connectors/tolerable_error_helper'
|
15
|
-
require 'core/filtering/validation_status'
|
10
|
+
require 'core/output_sink'
|
16
11
|
require 'utility'
|
17
|
-
require '
|
12
|
+
require 'app/config'
|
13
|
+
require 'active_support/core_ext/hash/indifferent_access'
|
18
14
|
|
19
15
|
module Connectors
|
20
16
|
module Base
|
@@ -36,50 +32,20 @@ module Connectors
|
|
36
32
|
raise 'Not implemented for this connector'
|
37
33
|
end
|
38
34
|
|
39
|
-
def self.kibana_features
|
40
|
-
[
|
41
|
-
Utility::Constants::FILTERING_RULES_FEATURE,
|
42
|
-
Utility::Constants::FILTERING_ADVANCED_FEATURE
|
43
|
-
]
|
44
|
-
end
|
45
|
-
|
46
|
-
def self.advanced_snippet_validator
|
47
|
-
AdvancedSnippetValidator
|
48
|
-
end
|
49
|
-
|
50
|
-
def self.validate_filtering(filtering = {})
|
51
|
-
# nothing to validate
|
52
|
-
return { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] } unless filtering.present?
|
53
|
-
|
54
|
-
filter = Utility::Filtering.extract_filter(filtering)
|
55
|
-
advanced_snippet = filter.dig(:advanced_snippet, :value)
|
56
|
-
|
57
|
-
snippet_validator_instance = advanced_snippet_validator.new(advanced_snippet)
|
58
|
-
|
59
|
-
snippet_validator_instance.is_snippet_valid?
|
60
|
-
end
|
61
|
-
|
62
35
|
attr_reader :rules, :advanced_filter_config
|
63
36
|
|
64
37
|
def initialize(configuration: {}, job_description: {})
|
65
|
-
error_monitor = Utility::ErrorMonitor.new
|
66
|
-
@tolerable_error_helper = Connectors::TolerableErrorHelper.new(error_monitor)
|
67
|
-
|
68
38
|
@configuration = configuration.dup || {}
|
69
39
|
@job_description = job_description&.dup || {}
|
70
40
|
|
71
|
-
|
41
|
+
filter = get_filter(@job_description[:filtering])
|
72
42
|
|
73
|
-
@rules =
|
74
|
-
@advanced_filter_config =
|
43
|
+
@rules = Utility::Common.return_if_present(filter[:rules], [])
|
44
|
+
@advanced_filter_config = Utility::Common.return_if_present(filter[:advanced_config], {})
|
75
45
|
end
|
76
46
|
|
77
47
|
def yield_documents; end
|
78
48
|
|
79
|
-
def yield_with_handling_tolerable_errors(identifier: nil, &block)
|
80
|
-
@tolerable_error_helper.yield_single_document(identifier: identifier, &block)
|
81
|
-
end
|
82
|
-
|
83
49
|
def do_health_check
|
84
50
|
raise 'Not implemented for this connector'
|
85
51
|
end
|
@@ -101,11 +67,16 @@ module Connectors
|
|
101
67
|
end
|
102
68
|
|
103
69
|
def filtering_present?
|
104
|
-
@advanced_filter_config.present?
|
70
|
+
@advanced_filter_config.present? || @rules.present?
|
105
71
|
end
|
106
72
|
|
107
|
-
|
108
|
-
|
73
|
+
private
|
74
|
+
|
75
|
+
def get_filter(filtering)
|
76
|
+
# assume for now, that first object in filtering array or a filter object itself is the only filtering object
|
77
|
+
filter = filtering.is_a?(Array) ? filtering[0] : filtering
|
78
|
+
|
79
|
+
filter.present? ? filter : {}
|
109
80
|
end
|
110
81
|
end
|
111
82
|
end
|
@@ -7,8 +7,6 @@
|
|
7
7
|
# frozen_string_literal: true
|
8
8
|
|
9
9
|
require 'connectors/base/connector'
|
10
|
-
require 'connectors/example/example_advanced_snippet_validator'
|
11
|
-
require 'core/filtering/validation_status'
|
12
10
|
require 'utility'
|
13
11
|
|
14
12
|
module Connectors
|
@@ -47,10 +45,6 @@ module Connectors
|
|
47
45
|
# raise 'something went wrong'
|
48
46
|
end
|
49
47
|
|
50
|
-
def self.advanced_snippet_validator
|
51
|
-
ExampleAdvancedSnippetValidator
|
52
|
-
end
|
53
|
-
|
54
48
|
def yield_documents
|
55
49
|
attachments = [
|
56
50
|
load_attachment('first_attachment.txt'),
|
@@ -11,8 +11,7 @@ require 'connectors/base/connector'
|
|
11
11
|
require 'connectors/gitlab/extractor'
|
12
12
|
require 'connectors/gitlab/custom_client'
|
13
13
|
require 'connectors/gitlab/adapter'
|
14
|
-
require '
|
15
|
-
require 'core/ingestion'
|
14
|
+
require 'core/output_sink'
|
16
15
|
|
17
16
|
module Connectors
|
18
17
|
module GitLab
|
@@ -37,10 +36,6 @@ module Connectors
|
|
37
36
|
}
|
38
37
|
end
|
39
38
|
|
40
|
-
def self.advanced_snippet_validator
|
41
|
-
GitLabAdvancedSnippetValidator
|
42
|
-
end
|
43
|
-
|
44
39
|
def initialize(configuration: {}, job_description: {})
|
45
40
|
super
|
46
41
|
|
@@ -7,9 +7,6 @@
|
|
7
7
|
# frozen_string_literal: true
|
8
8
|
|
9
9
|
require 'connectors/base/connector'
|
10
|
-
require 'core/filtering/validation_status'
|
11
|
-
require 'connectors/mongodb/mongo_rules_parser'
|
12
|
-
require 'connectors/mongodb/mongo_advanced_snippet_against_schema_validator'
|
13
10
|
require 'mongo'
|
14
11
|
require 'utility'
|
15
12
|
|
@@ -31,31 +28,27 @@ module Connectors
|
|
31
28
|
|
32
29
|
def self.configurable_fields
|
33
30
|
{
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
31
|
+
:host => {
|
32
|
+
:label => 'Server Hostname'
|
33
|
+
},
|
34
|
+
:user => {
|
35
|
+
:label => 'Username'
|
36
|
+
},
|
37
|
+
:password => {
|
38
|
+
:label => 'Password'
|
39
|
+
},
|
40
|
+
:database => {
|
41
|
+
:label => 'Database'
|
42
|
+
},
|
43
|
+
:collection => {
|
44
|
+
:label => 'Collection'
|
45
|
+
},
|
46
|
+
:direct_connection => {
|
47
|
+
:label => 'Direct connection? (true/false)'
|
48
|
+
}
|
52
49
|
}
|
53
50
|
end
|
54
51
|
|
55
|
-
def self.advanced_snippet_validator
|
56
|
-
MongoAdvancedSnippetAgainstSchemaValidator
|
57
|
-
end
|
58
|
-
|
59
52
|
def initialize(configuration: {}, job_description: {})
|
60
53
|
super
|
61
54
|
|
@@ -68,6 +61,8 @@ module Connectors
|
|
68
61
|
end
|
69
62
|
|
70
63
|
def yield_documents
|
64
|
+
check_filtering
|
65
|
+
|
71
66
|
with_client do |client|
|
72
67
|
# We do paging using skip().limit() here to make Ruby recycle the memory for each page pulled from the server after it's not needed any more.
|
73
68
|
# This gives us more control on the usage of the memory (we can adjust PAGE_SIZE constant for that to decrease max memory consumption).
|
@@ -94,15 +89,15 @@ module Connectors
|
|
94
89
|
loop do
|
95
90
|
found_in_page = 0
|
96
91
|
|
97
|
-
Utility::Logger.info("Requesting #{PAGE_SIZE} documents from MongoDB (Starting at #{skip})")
|
98
92
|
view = cursor.skip(skip).limit(PAGE_SIZE)
|
99
93
|
view.each do |document|
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
94
|
+
yield serialize(document)
|
95
|
+
|
96
|
+
found_in_page += 1
|
97
|
+
found_overall += 1
|
98
|
+
|
99
|
+
overall_limit_reached = found_overall >= overall_limit && overall_limit != Float::INFINITY
|
100
|
+
|
106
101
|
break if overall_limit_reached
|
107
102
|
end
|
108
103
|
|
@@ -122,11 +117,22 @@ module Connectors
|
|
122
117
|
|
123
118
|
return create_aggregate_cursor(collection) if @advanced_filter_config[:aggregate].present?
|
124
119
|
|
125
|
-
return create_simple_rules_cursor(collection) if @rules.present?
|
126
|
-
|
127
120
|
collection.find
|
128
121
|
end
|
129
122
|
|
123
|
+
def check_filtering
|
124
|
+
return unless filtering_present?
|
125
|
+
|
126
|
+
check_find_and_aggregate
|
127
|
+
end
|
128
|
+
|
129
|
+
def check_find_and_aggregate
|
130
|
+
if @advanced_filter_config.keys.size != 1
|
131
|
+
invalid_keys_msg = "Only one of #{ALLOWED_TOP_LEVEL_FILTER_KEYS} is allowed in the filtering object. Keys present: '#{@advanced_filter_config.keys}'."
|
132
|
+
raise Utility::InvalidFilterConfigError.new(invalid_keys_msg)
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
130
136
|
def create_aggregate_cursor(collection)
|
131
137
|
aggregate = @advanced_filter_config[:aggregate]
|
132
138
|
|
@@ -153,16 +159,6 @@ module Connectors
|
|
153
159
|
[collection.find(filter, options), options]
|
154
160
|
end
|
155
161
|
|
156
|
-
def create_simple_rules_cursor(collection)
|
157
|
-
filter = {}
|
158
|
-
if @rules.present?
|
159
|
-
parser = MongoRulesParser.new(@rules)
|
160
|
-
filter = parser.parse
|
161
|
-
end
|
162
|
-
Utility::Logger.info("Filtering with simple rules filter: #{filter}")
|
163
|
-
filter.present? ? collection.find(filter) : collection.find
|
164
|
-
end
|
165
|
-
|
166
162
|
def extract_options(mongodb_function)
|
167
163
|
mongodb_function[:options].present? ? mongodb_function[:options] : {}
|
168
164
|
end
|
@@ -177,9 +173,9 @@ module Connectors
|
|
177
173
|
raise "Invalid value for 'Direct connection' : #{@direct_connection}." unless %w[true false].include?(@direct_connection.to_s.strip.downcase)
|
178
174
|
|
179
175
|
args = {
|
180
|
-
|
181
|
-
|
182
|
-
|
176
|
+
database: @database,
|
177
|
+
direct_connection: to_boolean(@direct_connection)
|
178
|
+
}
|
183
179
|
|
184
180
|
if @user.present? || @password.present?
|
185
181
|
args[:user] = @user
|
data/lib/core/configuration.rb
CHANGED
@@ -24,10 +24,8 @@ module Core
|
|
24
24
|
return
|
25
25
|
end
|
26
26
|
configuration = connector_class.configurable_fields_indifferent_access
|
27
|
-
features = connector_class.kibana_features.each_with_object({}) { |feature, hsh| hsh[feature] = true }
|
28
27
|
doc = {
|
29
|
-
:configuration => configuration
|
30
|
-
:features => features
|
28
|
+
:configuration => configuration
|
31
29
|
}
|
32
30
|
|
33
31
|
doc[:service_type] = service_type if service_type && connector_settings.needs_service_type?
|
@@ -23,24 +23,24 @@ module Core
|
|
23
23
|
|
24
24
|
DEFAULT_PAGE_SIZE = 100
|
25
25
|
|
26
|
+
# Error Classes
|
27
|
+
class ConnectorNotFoundError < StandardError; end
|
28
|
+
|
26
29
|
def self.fetch_by_id(connector_id)
|
27
30
|
es_response = ElasticConnectorActions.get_connector(connector_id)
|
28
|
-
return nil unless es_response[:found]
|
29
|
-
|
30
31
|
connectors_meta = ElasticConnectorActions.connectors_meta
|
32
|
+
|
33
|
+
raise ConnectorNotFoundError.new("Connector with id=#{connector_id} was not found.") unless es_response[:found]
|
31
34
|
new(es_response, connectors_meta)
|
32
35
|
end
|
33
36
|
|
37
|
+
def initialize(es_response, connectors_meta)
|
38
|
+
@elasticsearch_response = es_response.with_indifferent_access
|
39
|
+
@connectors_meta = connectors_meta.with_indifferent_access
|
40
|
+
end
|
41
|
+
|
34
42
|
def self.fetch_native_connectors(page_size = DEFAULT_PAGE_SIZE)
|
35
|
-
|
36
|
-
query = {
|
37
|
-
bool: {
|
38
|
-
filter: [
|
39
|
-
{ term: { is_native: true } },
|
40
|
-
{ terms: { service_type: Connectors::REGISTRY.registered_connectors } }
|
41
|
-
]
|
42
|
-
}
|
43
|
-
}
|
43
|
+
query = { term: { is_native: true } }
|
44
44
|
fetch_connectors_by_query(query, page_size)
|
45
45
|
end
|
46
46
|
|
@@ -83,26 +83,23 @@ module Core
|
|
83
83
|
end
|
84
84
|
|
85
85
|
def filtering
|
86
|
-
|
87
|
-
filtering = @elasticsearch_response.dig(:_source, :filtering)
|
88
|
-
|
89
|
-
Utility::Filtering.extract_filter(filtering)
|
86
|
+
Utility::Common.return_if_present(@elasticsearch_response[:filtering], DEFAULT_FILTERING)
|
90
87
|
end
|
91
88
|
|
92
89
|
def request_pipeline
|
93
|
-
Utility::Common.return_if_present(@elasticsearch_response.dig(:
|
90
|
+
Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
|
94
91
|
end
|
95
92
|
|
96
93
|
def extract_binary_content?
|
97
|
-
Utility::Common.return_if_present(@elasticsearch_response.dig(:
|
94
|
+
Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
|
98
95
|
end
|
99
96
|
|
100
97
|
def reduce_whitespace?
|
101
|
-
Utility::Common.return_if_present(@elasticsearch_response.dig(:
|
98
|
+
Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
|
102
99
|
end
|
103
100
|
|
104
101
|
def run_ml_inference?
|
105
|
-
Utility::Common.return_if_present(@elasticsearch_response.dig(:
|
102
|
+
Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
|
106
103
|
end
|
107
104
|
|
108
105
|
def formatted
|
@@ -119,39 +116,6 @@ module Core
|
|
119
116
|
index_name&.start_with?(Utility::Constants::CONTENT_INDEX_PREFIX)
|
120
117
|
end
|
121
118
|
|
122
|
-
def ready_for_sync?
|
123
|
-
Connectors::REGISTRY.registered?(service_type) &&
|
124
|
-
valid_index_name? &&
|
125
|
-
connector_status_allows_sync?
|
126
|
-
end
|
127
|
-
|
128
|
-
def running?
|
129
|
-
@elasticsearch_response[:_source][:last_sync_status] == Connectors::SyncStatus::IN_PROGRESS
|
130
|
-
end
|
131
|
-
|
132
|
-
def update_last_sync!(job)
|
133
|
-
doc = {
|
134
|
-
:last_sync_status => job.status,
|
135
|
-
:last_synced => Time.now,
|
136
|
-
:last_sync_error => job.error,
|
137
|
-
:error => job.error
|
138
|
-
}
|
139
|
-
|
140
|
-
if job.terminated?
|
141
|
-
doc[:last_indexed_document_count] = job[:indexed_document_count]
|
142
|
-
doc[:last_deleted_document_count] = job[:deleted_document_count]
|
143
|
-
end
|
144
|
-
|
145
|
-
Core::ElasticConnectorActions.update_connector_fields(job.connector_id, doc)
|
146
|
-
end
|
147
|
-
|
148
|
-
private
|
149
|
-
|
150
|
-
def initialize(es_response, connectors_meta)
|
151
|
-
@elasticsearch_response = es_response.with_indifferent_access
|
152
|
-
@connectors_meta = connectors_meta.with_indifferent_access
|
153
|
-
end
|
154
|
-
|
155
119
|
def self.fetch_connectors_by_query(query, page_size)
|
156
120
|
connectors_meta = ElasticConnectorActions.connectors_meta
|
157
121
|
|