connectors_service 8.6.0.4.pre.20221116T024501Z → 8.6.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/config/connectors.yml +6 -6
- data/lib/app/app.rb +0 -4
- data/lib/app/dispatcher.rb +17 -42
- data/lib/app/preflight_check.rb +0 -11
- data/lib/connectors/base/connector.rb +14 -43
- data/lib/connectors/example/connector.rb +0 -6
- data/lib/connectors/gitlab/connector.rb +1 -6
- data/lib/connectors/mongodb/connector.rb +43 -47
- data/lib/connectors/sync_status.rb +1 -6
- data/lib/core/configuration.rb +1 -3
- data/lib/core/connector_settings.rb +16 -52
- data/lib/core/elastic_connector_actions.rb +59 -320
- data/lib/core/output_sink/base_sink.rb +33 -0
- data/lib/core/output_sink/combined_sink.rb +38 -0
- data/lib/core/output_sink/console_sink.rb +51 -0
- data/lib/core/output_sink/es_sink.rb +74 -0
- data/lib/core/{ingestion.rb → output_sink.rb} +5 -1
- data/lib/core/scheduler.rb +10 -40
- data/lib/core/single_scheduler.rb +1 -1
- data/lib/core/sync_job_runner.rb +16 -72
- data/lib/core.rb +0 -4
- data/lib/utility/constants.rb +0 -2
- data/lib/utility/errors.rb +12 -0
- data/lib/utility/logger.rb +1 -1
- data/lib/utility.rb +4 -11
- metadata +9 -27
- data/lib/connectors/base/advanced_snippet_against_schema_validator.rb +0 -173
- data/lib/connectors/base/advanced_snippet_validator.rb +0 -34
- data/lib/connectors/base/simple_rules_parser.rb +0 -42
- data/lib/connectors/example/example_advanced_snippet_validator.rb +0 -35
- data/lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb +0 -35
- data/lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb +0 -22
- data/lib/connectors/mongodb/mongo_advanced_snippet_schema.rb +0 -292
- data/lib/connectors/mongodb/mongo_rules_parser.rb +0 -81
- data/lib/connectors/tolerable_error_helper.rb +0 -43
- data/lib/core/connector_job.rb +0 -210
- data/lib/core/filtering/post_process_engine.rb +0 -39
- data/lib/core/filtering/post_process_result.rb +0 -27
- data/lib/core/filtering/simple_rule.rb +0 -141
- data/lib/core/filtering/validation_job_runner.rb +0 -53
- data/lib/core/filtering/validation_status.rb +0 -17
- data/lib/core/filtering.rb +0 -17
- data/lib/core/ingestion/es_sink.rb +0 -118
- data/lib/core/jobs/consumer.rb +0 -114
- data/lib/core/jobs/producer.rb +0 -26
- data/lib/utility/bulk_queue.rb +0 -85
- data/lib/utility/error_monitor.rb +0 -108
- data/lib/utility/filtering.rb +0 -22
data/lib/core/scheduler.rb
CHANGED
@@ -10,7 +10,6 @@ require 'time'
|
|
10
10
|
require 'fugit'
|
11
11
|
require 'core/connector_settings'
|
12
12
|
require 'core/elastic_connector_actions'
|
13
|
-
require 'core/filtering/validation_status'
|
14
13
|
require 'utility/cron'
|
15
14
|
require 'utility/logger'
|
16
15
|
require 'utility/exception_tracking'
|
@@ -39,18 +38,15 @@ module Core
|
|
39
38
|
if configuration_triggered?(cs)
|
40
39
|
yield cs, :configuration
|
41
40
|
end
|
42
|
-
|
43
|
-
|
44
|
-
|
41
|
+
end
|
42
|
+
if @is_shutting_down
|
43
|
+
break
|
45
44
|
end
|
46
45
|
rescue *Utility::AUTHORIZATION_ERRORS => e
|
47
46
|
Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
|
48
47
|
rescue StandardError => e
|
49
48
|
Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
|
50
49
|
ensure
|
51
|
-
if @is_shutting_down
|
52
|
-
break
|
53
|
-
end
|
54
50
|
if @poll_interval > 0 && !@is_shutting_down
|
55
51
|
Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
|
56
52
|
sleep(@poll_interval)
|
@@ -66,6 +62,8 @@ module Core
|
|
66
62
|
private
|
67
63
|
|
68
64
|
def sync_triggered?(connector_settings)
|
65
|
+
return false unless connector_registered?(connector_settings.service_type)
|
66
|
+
|
69
67
|
unless connector_settings.valid_index_name?
|
70
68
|
Utility::Logger.warn("The index name of #{connector_settings.formatted} is invalid.")
|
71
69
|
return false
|
@@ -131,6 +129,8 @@ module Core
|
|
131
129
|
end
|
132
130
|
|
133
131
|
def heartbeat_triggered?(connector_settings)
|
132
|
+
return false unless connector_registered?(connector_settings.service_type)
|
133
|
+
|
134
134
|
last_seen = connector_settings[:last_seen]
|
135
135
|
return true if last_seen.nil? || last_seen.empty?
|
136
136
|
last_seen = begin
|
@@ -144,41 +144,11 @@ module Core
|
|
144
144
|
end
|
145
145
|
|
146
146
|
def configuration_triggered?(connector_settings)
|
147
|
-
connector_settings.needs_service_type? || connector_settings.
|
148
|
-
|
149
|
-
|
150
|
-
def filtering_validation_triggered?(connector_settings)
|
151
|
-
filtering = connector_settings.filtering
|
152
|
-
|
153
|
-
unless filtering.present?
|
154
|
-
Utility::Logger.debug("#{connector_settings.formatted} does not contain filtering to be validated.")
|
155
|
-
|
156
|
-
return false
|
147
|
+
if connector_settings.needs_service_type? || connector_registered?(connector_settings.service_type)
|
148
|
+
return connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
|
157
149
|
end
|
158
150
|
|
159
|
-
|
160
|
-
|
161
|
-
unless draft_filters.present?
|
162
|
-
Utility::Logger.debug("#{connector_settings.formatted} does not contain a draft filter to be validated.")
|
163
|
-
|
164
|
-
return false
|
165
|
-
end
|
166
|
-
|
167
|
-
validation = draft_filters[:validation]
|
168
|
-
|
169
|
-
unless validation.present?
|
170
|
-
Utility::Logger.warn("#{connector_settings.formatted} does not contain a validation object inside draft filtering. Check connectors index.")
|
171
|
-
|
172
|
-
return false
|
173
|
-
end
|
174
|
-
|
175
|
-
unless validation[:state] == Core::Filtering::ValidationStatus::EDITED
|
176
|
-
Utility::Logger.debug("#{connector_settings.formatted} filtering validation needs to be in state #{Core::Filtering::ValidationStatus::EDITED} to be able to validate it.")
|
177
|
-
|
178
|
-
return false
|
179
|
-
end
|
180
|
-
|
181
|
-
true
|
151
|
+
false
|
182
152
|
end
|
183
153
|
|
184
154
|
def connector_registered?(service_type)
|
@@ -20,7 +20,7 @@ module Core
|
|
20
20
|
|
21
21
|
def connector_settings
|
22
22
|
connector_settings = Core::ConnectorSettings.fetch_by_id(@connector_id)
|
23
|
-
[connector_settings]
|
23
|
+
[connector_settings]
|
24
24
|
rescue *Utility::AUTHORIZATION_ERRORS => e
|
25
25
|
# should be handled by the general scheduler
|
26
26
|
raise e
|
data/lib/core/sync_job_runner.rb
CHANGED
@@ -8,9 +8,7 @@
|
|
8
8
|
|
9
9
|
require 'connectors/connector_status'
|
10
10
|
require 'connectors/registry'
|
11
|
-
require 'core/
|
12
|
-
require 'core/ingestion'
|
13
|
-
require 'core/filtering/validation_status'
|
11
|
+
require 'core/output_sink'
|
14
12
|
require 'utility'
|
15
13
|
|
16
14
|
module Core
|
@@ -21,21 +19,16 @@ module Core
|
|
21
19
|
end
|
22
20
|
|
23
21
|
class SyncJobRunner
|
24
|
-
|
25
|
-
|
26
|
-
def initialize(connector_settings, job)
|
22
|
+
def initialize(connector_settings)
|
27
23
|
@connector_settings = connector_settings
|
28
|
-
@sink = Core::
|
24
|
+
@sink = Core::OutputSink::EsSink.new(connector_settings.index_name, @connector_settings.request_pipeline)
|
29
25
|
@connector_class = Connectors::REGISTRY.connector_class(connector_settings.service_type)
|
30
26
|
@sync_finished = false
|
31
|
-
@sync_error = nil
|
32
27
|
@status = {
|
33
28
|
:indexed_document_count => 0,
|
34
29
|
:deleted_document_count => 0,
|
35
|
-
:indexed_document_volume => 0,
|
36
30
|
:error => nil
|
37
31
|
}
|
38
|
-
@job = job
|
39
32
|
end
|
40
33
|
|
41
34
|
def execute
|
@@ -48,17 +41,8 @@ module Core
|
|
48
41
|
def do_sync!
|
49
42
|
Utility::Logger.info("Claiming a sync job for connector #{@connector_settings.id}.")
|
50
43
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
Core::ElasticConnectorActions.update_connector_last_sync_status(@connector_settings.id, Connectors::SyncStatus::IN_PROGRESS)
|
55
|
-
|
56
|
-
# claim the job
|
57
|
-
@job.make_running!
|
58
|
-
|
59
|
-
job_description = @job.es_source
|
60
|
-
job_id = @job.id
|
61
|
-
job_description['_id'] = job_id
|
44
|
+
job_description = ElasticConnectorActions.claim_job(@connector_settings.id)
|
45
|
+
job_id = job_description['_id']
|
62
46
|
|
63
47
|
unless job_id.present?
|
64
48
|
Utility::Logger.error("Failed to claim the job for #{@connector_settings.id}. Please check the logs for the cause of this error.")
|
@@ -68,10 +52,6 @@ module Core
|
|
68
52
|
begin
|
69
53
|
Utility::Logger.debug("Successfully claimed job for connector #{@connector_settings.id}.")
|
70
54
|
|
71
|
-
Utility::Logger.info("Checking active filtering for sync job #{job_id} for connector #{@connector_settings.id}.")
|
72
|
-
validate_filtering(job_description.dig(:connector, :filtering))
|
73
|
-
Utility::Logger.debug("Active filtering for sync job #{job_id} for connector #{@connector_settings.id} is valid.")
|
74
|
-
|
75
55
|
connector_instance = Connectors::REGISTRY.connector(@connector_settings.service_type, @connector_settings.configuration, job_description: job_description)
|
76
56
|
|
77
57
|
connector_instance.do_health_check!
|
@@ -81,21 +61,11 @@ module Core
|
|
81
61
|
|
82
62
|
Utility::Logger.debug("#{existing_ids.size} documents are present in index #{@connector_settings.index_name}.")
|
83
63
|
|
84
|
-
post_processing_engine = Core::Filtering::PostProcessEngine.new(job_description)
|
85
|
-
reporting_cycle_start = Time.now
|
86
|
-
Utility::Logger.info('Yielding documents...')
|
87
64
|
connector_instance.yield_documents do |document|
|
88
65
|
document = add_ingest_metadata(document)
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
incoming_ids << document['id']
|
93
|
-
end
|
94
|
-
|
95
|
-
if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
|
96
|
-
ElasticConnectorActions.update_sync(job_id, @sink.ingestion_stats.merge(:metadata => connector_instance.metadata))
|
97
|
-
reporting_cycle_start = Time.now
|
98
|
-
end
|
66
|
+
@sink.ingest(document)
|
67
|
+
incoming_ids << document['id']
|
68
|
+
@status[:indexed_document_count] += 1
|
99
69
|
end
|
100
70
|
|
101
71
|
ids_to_delete = existing_ids - incoming_ids.uniq
|
@@ -104,11 +74,7 @@ module Core
|
|
104
74
|
|
105
75
|
ids_to_delete.each do |id|
|
106
76
|
@sink.delete(id)
|
107
|
-
|
108
|
-
if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
|
109
|
-
ElasticConnectorActions.update_sync(job_id, @sink.ingestion_stats.merge(:metadata => connector_instance.metadata))
|
110
|
-
reporting_cycle_start = Time.now
|
111
|
-
end
|
77
|
+
@status[:deleted_document_count] += 1
|
112
78
|
end
|
113
79
|
|
114
80
|
@sink.flush
|
@@ -117,34 +83,22 @@ module Core
|
|
117
83
|
# occurred as most of the time the main execution thread is interrupted and we miss this Signal/Exception here
|
118
84
|
@sync_finished = true
|
119
85
|
rescue StandardError => e
|
120
|
-
@
|
86
|
+
@status[:error] = e.message
|
121
87
|
Utility::ExceptionTracking.log_exception(e)
|
88
|
+
ElasticConnectorActions.update_connector_status(@connector_settings.id, Connectors::ConnectorStatus::ERROR, Utility::Logger.abbreviated_message(e.message))
|
122
89
|
ensure
|
123
|
-
stats = @sink.ingestion_stats
|
124
|
-
|
125
|
-
Utility::Logger.debug("Sync stats are: #{stats}")
|
126
|
-
|
127
|
-
@status[:indexed_document_count] = stats[:indexed_document_count]
|
128
|
-
@status[:deleted_document_count] = stats[:deleted_document_count]
|
129
|
-
@status[:indexed_document_volume] = stats[:indexed_document_volume]
|
130
|
-
|
131
90
|
Utility::Logger.info("Upserted #{@status[:indexed_document_count]} documents into #{@connector_settings.index_name}.")
|
132
91
|
Utility::Logger.info("Deleted #{@status[:deleted_document_count]} documents into #{@connector_settings.index_name}.")
|
133
92
|
|
134
93
|
# Make sure to not override a previous error message
|
135
|
-
if !@sync_finished && @
|
136
|
-
@
|
137
|
-
end
|
138
|
-
|
139
|
-
unless connector_instance.nil?
|
140
|
-
metadata = @sink.ingestion_stats.merge(:metadata => connector_instance.metadata)
|
141
|
-
metadata[:total_document_count] = ElasticConnectorActions.document_count(@connector_settings.index_name)
|
94
|
+
if !@sync_finished && @status[:error].nil?
|
95
|
+
@status[:error] = 'Sync thread didn\'t finish execution. Check connector logs for more details.'
|
142
96
|
end
|
143
97
|
|
144
|
-
ElasticConnectorActions.complete_sync(@connector_settings.id, job_id,
|
98
|
+
ElasticConnectorActions.complete_sync(@connector_settings.id, job_id, @status.dup)
|
145
99
|
|
146
|
-
if @
|
147
|
-
Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@
|
100
|
+
if @status[:error]
|
101
|
+
Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@status[:error]}'.")
|
148
102
|
else
|
149
103
|
Utility::Logger.info("Successfully synced for connector #{@connector_settings.id}.")
|
150
104
|
end
|
@@ -165,15 +119,5 @@ module Core
|
|
165
119
|
|
166
120
|
raise IncompatibleConfigurableFieldsError.new(@connector_class.service_type, expected_fields, actual_fields) if expected_fields != actual_fields
|
167
121
|
end
|
168
|
-
|
169
|
-
def validate_filtering(filtering)
|
170
|
-
validation_result = @connector_class.validate_filtering(filtering)
|
171
|
-
|
172
|
-
wrong_state_error = Utility::InvalidFilterConfigError.new("Active filtering is not in valid state (current state: #{validation_result[:state]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
|
173
|
-
raise wrong_state_error if validation_result[:state] != Core::Filtering::ValidationStatus::VALID
|
174
|
-
|
175
|
-
errors_present_error = Utility::InvalidFilterConfigError.new("Active filtering is in valid state, but errors were detected (errors: #{validation_result[:errors]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
|
176
|
-
raise errors_present_error if validation_result[:errors].present?
|
177
|
-
end
|
178
122
|
end
|
179
123
|
end
|
data/lib/core.rb
CHANGED
@@ -7,14 +7,10 @@
|
|
7
7
|
# frozen_string_literal: true
|
8
8
|
|
9
9
|
require 'core/configuration'
|
10
|
-
require 'core/connector_job'
|
11
10
|
require 'core/connector_settings'
|
12
11
|
require 'core/elastic_connector_actions'
|
13
|
-
require 'core/filtering'
|
14
12
|
require 'core/heartbeat'
|
15
13
|
require 'core/scheduler'
|
16
14
|
require 'core/single_scheduler'
|
17
15
|
require 'core/native_scheduler'
|
18
16
|
require 'core/sync_job_runner'
|
19
|
-
require 'core/jobs/producer'
|
20
|
-
require 'core/jobs/consumer'
|
data/lib/utility/constants.rb
CHANGED
data/lib/utility/errors.rb
CHANGED
@@ -60,6 +60,18 @@ module Utility
|
|
60
60
|
class JobDocumentLimitError < StandardError; end
|
61
61
|
class JobClaimingError < StandardError; end
|
62
62
|
|
63
|
+
class MonitoringError < StandardError
|
64
|
+
attr_accessor :tripped_by
|
65
|
+
|
66
|
+
def initialize(message = nil, tripped_by: nil)
|
67
|
+
super("#{message}#{tripped_by.present? ? " Tripped by - #{tripped_by.class}: #{tripped_by.message}" : ''}")
|
68
|
+
@tripped_by = tripped_by
|
69
|
+
end
|
70
|
+
end
|
71
|
+
class MaxSuccessiveErrorsExceededError < MonitoringError; end
|
72
|
+
class MaxErrorsExceededError < MonitoringError; end
|
73
|
+
class MaxErrorsInWindowExceededError < MonitoringError; end
|
74
|
+
|
63
75
|
class JobSyncNotPossibleYetError < StandardError
|
64
76
|
attr_accessor :sync_will_be_possible_at
|
65
77
|
|
data/lib/utility/logger.rb
CHANGED
@@ -23,7 +23,7 @@ module Utility
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def logger
|
26
|
-
@logger ||=
|
26
|
+
@logger ||= Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
|
27
27
|
end
|
28
28
|
|
29
29
|
SUPPORTED_LOG_LEVELS.each do |level|
|
data/lib/utility.rb
CHANGED
@@ -4,21 +4,14 @@
|
|
4
4
|
# you may not use this file except in compliance with the Elastic License.
|
5
5
|
#
|
6
6
|
|
7
|
-
# !!!!!!!!
|
8
|
-
# IF YOU EDIT THIS FILE, YOU MUST EDIT THE `connectors_utility.gemspec`
|
9
|
-
require 'utility/bulk_queue'
|
10
|
-
require 'utility/common'
|
11
7
|
require 'utility/constants'
|
12
8
|
require 'utility/cron'
|
13
|
-
require 'utility/
|
14
|
-
require 'utility/elasticsearch/index/text_analysis_settings'
|
15
|
-
require 'utility/environment'
|
16
|
-
require 'utility/error_monitor'
|
9
|
+
require 'utility/common'
|
17
10
|
require 'utility/errors'
|
18
|
-
require 'utility/filtering'
|
19
11
|
require 'utility/es_client'
|
12
|
+
require 'utility/environment'
|
20
13
|
require 'utility/exception_tracking'
|
21
14
|
require 'utility/extension_mapping_util'
|
22
15
|
require 'utility/logger'
|
23
|
-
|
24
|
-
|
16
|
+
require 'utility/elasticsearch/index/mappings'
|
17
|
+
require 'utility/elasticsearch/index/text_analysis_settings'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: connectors_service
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 8.6.0.4
|
4
|
+
version: 8.6.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-11-
|
11
|
+
date: 2022-11-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -396,49 +396,34 @@ files:
|
|
396
396
|
- lib/app/version.rb
|
397
397
|
- lib/connectors.rb
|
398
398
|
- lib/connectors/base/adapter.rb
|
399
|
-
- lib/connectors/base/advanced_snippet_against_schema_validator.rb
|
400
|
-
- lib/connectors/base/advanced_snippet_validator.rb
|
401
399
|
- lib/connectors/base/connector.rb
|
402
400
|
- lib/connectors/base/custom_client.rb
|
403
|
-
- lib/connectors/base/simple_rules_parser.rb
|
404
401
|
- lib/connectors/connector_status.rb
|
405
402
|
- lib/connectors/crawler/scheduler.rb
|
406
403
|
- lib/connectors/example/attachments/first_attachment.txt
|
407
404
|
- lib/connectors/example/attachments/second_attachment.txt
|
408
405
|
- lib/connectors/example/attachments/third_attachment.txt
|
409
406
|
- lib/connectors/example/connector.rb
|
410
|
-
- lib/connectors/example/example_advanced_snippet_validator.rb
|
411
407
|
- lib/connectors/gitlab/adapter.rb
|
412
408
|
- lib/connectors/gitlab/connector.rb
|
413
409
|
- lib/connectors/gitlab/custom_client.rb
|
414
410
|
- lib/connectors/gitlab/extractor.rb
|
415
|
-
- lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb
|
416
411
|
- lib/connectors/mongodb/connector.rb
|
417
|
-
- lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb
|
418
|
-
- lib/connectors/mongodb/mongo_advanced_snippet_schema.rb
|
419
|
-
- lib/connectors/mongodb/mongo_rules_parser.rb
|
420
412
|
- lib/connectors/registry.rb
|
421
413
|
- lib/connectors/sync_status.rb
|
422
|
-
- lib/connectors/tolerable_error_helper.rb
|
423
414
|
- lib/connectors_service.rb
|
424
415
|
- lib/connectors_utility.rb
|
425
416
|
- lib/core.rb
|
426
417
|
- lib/core/configuration.rb
|
427
|
-
- lib/core/connector_job.rb
|
428
418
|
- lib/core/connector_settings.rb
|
429
419
|
- lib/core/elastic_connector_actions.rb
|
430
|
-
- lib/core/filtering.rb
|
431
|
-
- lib/core/filtering/post_process_engine.rb
|
432
|
-
- lib/core/filtering/post_process_result.rb
|
433
|
-
- lib/core/filtering/simple_rule.rb
|
434
|
-
- lib/core/filtering/validation_job_runner.rb
|
435
|
-
- lib/core/filtering/validation_status.rb
|
436
420
|
- lib/core/heartbeat.rb
|
437
|
-
- lib/core/ingestion.rb
|
438
|
-
- lib/core/ingestion/es_sink.rb
|
439
|
-
- lib/core/jobs/consumer.rb
|
440
|
-
- lib/core/jobs/producer.rb
|
441
421
|
- lib/core/native_scheduler.rb
|
422
|
+
- lib/core/output_sink.rb
|
423
|
+
- lib/core/output_sink/base_sink.rb
|
424
|
+
- lib/core/output_sink/combined_sink.rb
|
425
|
+
- lib/core/output_sink/console_sink.rb
|
426
|
+
- lib/core/output_sink/es_sink.rb
|
442
427
|
- lib/core/scheduler.rb
|
443
428
|
- lib/core/single_scheduler.rb
|
444
429
|
- lib/core/sync_job_runner.rb
|
@@ -447,7 +432,6 @@ files:
|
|
447
432
|
- lib/stubs/connectors/stats.rb
|
448
433
|
- lib/stubs/service_type.rb
|
449
434
|
- lib/utility.rb
|
450
|
-
- lib/utility/bulk_queue.rb
|
451
435
|
- lib/utility/common.rb
|
452
436
|
- lib/utility/constants.rb
|
453
437
|
- lib/utility/cron.rb
|
@@ -455,12 +439,10 @@ files:
|
|
455
439
|
- lib/utility/elasticsearch/index/mappings.rb
|
456
440
|
- lib/utility/elasticsearch/index/text_analysis_settings.rb
|
457
441
|
- lib/utility/environment.rb
|
458
|
-
- lib/utility/error_monitor.rb
|
459
442
|
- lib/utility/errors.rb
|
460
443
|
- lib/utility/es_client.rb
|
461
444
|
- lib/utility/exception_tracking.rb
|
462
445
|
- lib/utility/extension_mapping_util.rb
|
463
|
-
- lib/utility/filtering.rb
|
464
446
|
- lib/utility/logger.rb
|
465
447
|
- lib/utility/middleware/basic_auth.rb
|
466
448
|
- lib/utility/middleware/bearer_auth.rb
|
@@ -480,9 +462,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
480
462
|
version: '0'
|
481
463
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
482
464
|
requirements:
|
483
|
-
- - "
|
465
|
+
- - ">="
|
484
466
|
- !ruby/object:Gem::Version
|
485
|
-
version:
|
467
|
+
version: '0'
|
486
468
|
requirements: []
|
487
469
|
rubygems_version: 3.0.3.1
|
488
470
|
signing_key:
|
@@ -1,173 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
3
|
-
# or more contributor license agreements. Licensed under the Elastic License;
|
4
|
-
# you may not use this file except in compliance with the Elastic License.
|
5
|
-
#
|
6
|
-
# frozen_string_literal: true
|
7
|
-
|
8
|
-
require 'active_support/core_ext/hash'
|
9
|
-
require 'utility/logger'
|
10
|
-
require 'connectors/base/advanced_snippet_validator'
|
11
|
-
require 'core/filtering/validation_status'
|
12
|
-
|
13
|
-
module Connectors
|
14
|
-
module Base
|
15
|
-
class AdvancedSnippetAgainstSchemaValidator < Connectors::Base::AdvancedSnippetValidator
|
16
|
-
|
17
|
-
MAX_RECURSION_DEPTH = 50
|
18
|
-
ADVANCED_SNIPPET_ID = 'advanced_snippet'
|
19
|
-
|
20
|
-
def initialize(advanced_snippet, schema)
|
21
|
-
super(advanced_snippet)
|
22
|
-
@schema = schema
|
23
|
-
end
|
24
|
-
|
25
|
-
def is_snippet_valid?
|
26
|
-
validation_result = validate_against_schema(@schema, @advanced_snippet)
|
27
|
-
log_validation_result(validation_result)
|
28
|
-
validation_result
|
29
|
-
end
|
30
|
-
|
31
|
-
private
|
32
|
-
|
33
|
-
def validate_against_schema(config_schema, advanced_snippet, recursion_depth = 0)
|
34
|
-
# Prevent unintentional/intentional SystemStackErrors/crashes
|
35
|
-
return unexpected_error if exceeded_recursion_depth?(recursion_depth)
|
36
|
-
|
37
|
-
return valid_snippet if config_schema.nil? || config_schema.empty?
|
38
|
-
|
39
|
-
schema_fields = config_schema[:fields].is_a?(Hash) ? config_schema.dig(:fields, :values) : config_schema[:fields]
|
40
|
-
snippet_field_names = advanced_snippet&.keys&.map(&:to_s)
|
41
|
-
schema_field_names = schema_fields.map { |field| field[:name] }
|
42
|
-
|
43
|
-
return unexpected_field(schema_field_names, snippet_field_names) if unexpected_field_present?(snippet_field_names, schema_field_names)
|
44
|
-
|
45
|
-
return fields_constraint_violation(config_schema[:fields]) if fields_constraints_violated?(config_schema, advanced_snippet)
|
46
|
-
|
47
|
-
schema_fields.each do |field|
|
48
|
-
name = field[:name]
|
49
|
-
type = field[:type]
|
50
|
-
optional = field[:optional] || false
|
51
|
-
|
52
|
-
snippet_field_value = advanced_snippet.with_indifferent_access[name]
|
53
|
-
|
54
|
-
next if optional && (snippet_field_value.nil? || !snippet_field_value.present?)
|
55
|
-
|
56
|
-
return wrong_names(snippet_field_names, name) unless snippet_field_names.include?(name)
|
57
|
-
|
58
|
-
return wrong_type(name, type, snippet_field_value) if type_error_present?(type, snippet_field_value)
|
59
|
-
|
60
|
-
if field[:fields].present?
|
61
|
-
validation_result = validate_against_schema(field, snippet_field_value, recursion_depth + 1)
|
62
|
-
|
63
|
-
return validation_result unless validation_result[:is_valid]
|
64
|
-
end
|
65
|
-
end
|
66
|
-
|
67
|
-
valid_snippet
|
68
|
-
end
|
69
|
-
|
70
|
-
def fields_constraints_violated?(config_schema, advanced_snippet)
|
71
|
-
return false unless config_schema[:fields].is_a?(Hash)
|
72
|
-
|
73
|
-
constraints = config_schema.dig(:fields, :constraints)
|
74
|
-
constraints = constraints.is_a?(Array) ? constraints : [constraints]
|
75
|
-
|
76
|
-
constraints.each do |constraint|
|
77
|
-
return true unless constraint.call(advanced_snippet)
|
78
|
-
end
|
79
|
-
|
80
|
-
false
|
81
|
-
end
|
82
|
-
|
83
|
-
def type_error_present?(schema_type, snippet_value)
|
84
|
-
return !schema_type.call(snippet_value) if schema_type.is_a?(Proc)
|
85
|
-
|
86
|
-
!snippet_value.is_a?(schema_type)
|
87
|
-
end
|
88
|
-
|
89
|
-
def exceeded_recursion_depth?(recursion_depth)
|
90
|
-
if recursion_depth >= MAX_RECURSION_DEPTH
|
91
|
-
Utility::Logger.warn("Recursion depth for filtering validation exceeded. (Max recursion depth: #{MAX_RECURSION_DEPTH})")
|
92
|
-
return true
|
93
|
-
end
|
94
|
-
|
95
|
-
false
|
96
|
-
end
|
97
|
-
|
98
|
-
def unexpected_field_present?(actual_field_names, expected_field_names)
|
99
|
-
difference = actual_field_names - expected_field_names
|
100
|
-
|
101
|
-
# we have field names, which we didn't expect
|
102
|
-
!difference.empty?
|
103
|
-
end
|
104
|
-
|
105
|
-
def valid_snippet
|
106
|
-
{
|
107
|
-
:state => Core::Filtering::ValidationStatus::VALID,
|
108
|
-
:errors => []
|
109
|
-
}
|
110
|
-
end
|
111
|
-
|
112
|
-
def unexpected_field(expected_fields, actual_fields)
|
113
|
-
{
|
114
|
-
:state => Core::Filtering::ValidationStatus::INVALID,
|
115
|
-
:errors => [
|
116
|
-
{
|
117
|
-
:ids => [ADVANCED_SNIPPET_ID],
|
118
|
-
:messages => ["Encountered unexpected fields '#{actual_fields}'. Expected: '#{expected_fields}'."]
|
119
|
-
}
|
120
|
-
]
|
121
|
-
}
|
122
|
-
end
|
123
|
-
|
124
|
-
def wrong_type(field_name, expected_type, actual_value)
|
125
|
-
{
|
126
|
-
:state => Core::Filtering::ValidationStatus::INVALID,
|
127
|
-
:errors => [
|
128
|
-
{
|
129
|
-
:ids => [ADVANCED_SNIPPET_ID],
|
130
|
-
:messages => ["Expected field type '#{expected_type.is_a?(Proc) ? 'custom matcher' : expected_type}' for field '#{field_name}', but got value '#{actual_value.inspect}' of type '#{actual_value.class}'."]
|
131
|
-
}
|
132
|
-
]
|
133
|
-
}
|
134
|
-
end
|
135
|
-
|
136
|
-
def wrong_names(actual_field_names, expected_field_name)
|
137
|
-
{
|
138
|
-
:state => Core::Filtering::ValidationStatus::INVALID,
|
139
|
-
:errors => [
|
140
|
-
{
|
141
|
-
:ids => [ADVANCED_SNIPPET_ID],
|
142
|
-
:messages => ["Expected field name '#{expected_field_name}', but got #{actual_field_names}."]
|
143
|
-
}
|
144
|
-
]
|
145
|
-
}
|
146
|
-
end
|
147
|
-
|
148
|
-
def fields_constraint_violation(fields)
|
149
|
-
{
|
150
|
-
:state => Core::Filtering::ValidationStatus::INVALID,
|
151
|
-
:errors => [
|
152
|
-
{
|
153
|
-
:ids => [ADVANCED_SNIPPET_ID],
|
154
|
-
:messages => ["A fields constraint was violated for fields: '#{fields[:values].map { |v| v[:name] }}'. Check advanced snippet field constraints."]
|
155
|
-
}
|
156
|
-
]
|
157
|
-
}
|
158
|
-
end
|
159
|
-
|
160
|
-
def unexpected_error
|
161
|
-
{
|
162
|
-
:state => Core::Filtering::ValidationStatus::INVALID,
|
163
|
-
:errors => [
|
164
|
-
{
|
165
|
-
:ids => [ADVANCED_SNIPPET_ID],
|
166
|
-
:messages => ['Unexpected error. Check logs for details.']
|
167
|
-
}
|
168
|
-
]
|
169
|
-
}
|
170
|
-
end
|
171
|
-
end
|
172
|
-
end
|
173
|
-
end
|
@@ -1,34 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
3
|
-
# or more contributor license agreements. Licensed under the Elastic License;
|
4
|
-
# you may not use this file except in compliance with the Elastic License.
|
5
|
-
#
|
6
|
-
# frozen_string_literal: true
|
7
|
-
|
8
|
-
require 'utility/logger'
|
9
|
-
|
10
|
-
module Connectors
|
11
|
-
module Base
|
12
|
-
class AdvancedSnippetValidator
|
13
|
-
|
14
|
-
def initialize(advanced_snippet)
|
15
|
-
@advanced_snippet = advanced_snippet || {}
|
16
|
-
end
|
17
|
-
|
18
|
-
def is_snippet_valid?
|
19
|
-
raise 'Advanced Snippet validation not implemented'
|
20
|
-
end
|
21
|
-
|
22
|
-
private
|
23
|
-
|
24
|
-
def log_validation_result(validation_result)
|
25
|
-
Utility::Logger.info("Filtering Advanced Configuration validation result: #{validation_result[:state]}")
|
26
|
-
if validation_result[:errors].present?
|
27
|
-
validation_result[:errors].each do |error|
|
28
|
-
Utility::Logger.warn("Validation error for: '#{error[:ids]}': '#{error[:messages]}'")
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|