connectors_service 8.6.0.4.pre.20221116T024501Z → 8.6.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/config/connectors.yml +6 -6
- data/lib/app/app.rb +0 -4
- data/lib/app/dispatcher.rb +17 -42
- data/lib/app/preflight_check.rb +0 -11
- data/lib/connectors/base/connector.rb +14 -43
- data/lib/connectors/example/connector.rb +0 -6
- data/lib/connectors/gitlab/connector.rb +1 -6
- data/lib/connectors/mongodb/connector.rb +43 -47
- data/lib/connectors/sync_status.rb +1 -6
- data/lib/core/configuration.rb +1 -3
- data/lib/core/connector_settings.rb +16 -52
- data/lib/core/elastic_connector_actions.rb +59 -320
- data/lib/core/output_sink/base_sink.rb +33 -0
- data/lib/core/output_sink/combined_sink.rb +38 -0
- data/lib/core/output_sink/console_sink.rb +51 -0
- data/lib/core/output_sink/es_sink.rb +74 -0
- data/lib/core/{ingestion.rb → output_sink.rb} +5 -1
- data/lib/core/scheduler.rb +10 -40
- data/lib/core/single_scheduler.rb +1 -1
- data/lib/core/sync_job_runner.rb +16 -72
- data/lib/core.rb +0 -4
- data/lib/utility/constants.rb +0 -2
- data/lib/utility/errors.rb +12 -0
- data/lib/utility/logger.rb +1 -1
- data/lib/utility.rb +4 -11
- metadata +9 -27
- data/lib/connectors/base/advanced_snippet_against_schema_validator.rb +0 -173
- data/lib/connectors/base/advanced_snippet_validator.rb +0 -34
- data/lib/connectors/base/simple_rules_parser.rb +0 -42
- data/lib/connectors/example/example_advanced_snippet_validator.rb +0 -35
- data/lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb +0 -35
- data/lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb +0 -22
- data/lib/connectors/mongodb/mongo_advanced_snippet_schema.rb +0 -292
- data/lib/connectors/mongodb/mongo_rules_parser.rb +0 -81
- data/lib/connectors/tolerable_error_helper.rb +0 -43
- data/lib/core/connector_job.rb +0 -210
- data/lib/core/filtering/post_process_engine.rb +0 -39
- data/lib/core/filtering/post_process_result.rb +0 -27
- data/lib/core/filtering/simple_rule.rb +0 -141
- data/lib/core/filtering/validation_job_runner.rb +0 -53
- data/lib/core/filtering/validation_status.rb +0 -17
- data/lib/core/filtering.rb +0 -17
- data/lib/core/ingestion/es_sink.rb +0 -118
- data/lib/core/jobs/consumer.rb +0 -114
- data/lib/core/jobs/producer.rb +0 -26
- data/lib/utility/bulk_queue.rb +0 -85
- data/lib/utility/error_monitor.rb +0 -108
- data/lib/utility/filtering.rb +0 -22
data/lib/core/scheduler.rb
CHANGED
@@ -10,7 +10,6 @@ require 'time'
|
|
10
10
|
require 'fugit'
|
11
11
|
require 'core/connector_settings'
|
12
12
|
require 'core/elastic_connector_actions'
|
13
|
-
require 'core/filtering/validation_status'
|
14
13
|
require 'utility/cron'
|
15
14
|
require 'utility/logger'
|
16
15
|
require 'utility/exception_tracking'
|
@@ -39,18 +38,15 @@ module Core
|
|
39
38
|
if configuration_triggered?(cs)
|
40
39
|
yield cs, :configuration
|
41
40
|
end
|
42
|
-
|
43
|
-
|
44
|
-
|
41
|
+
end
|
42
|
+
if @is_shutting_down
|
43
|
+
break
|
45
44
|
end
|
46
45
|
rescue *Utility::AUTHORIZATION_ERRORS => e
|
47
46
|
Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
|
48
47
|
rescue StandardError => e
|
49
48
|
Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
|
50
49
|
ensure
|
51
|
-
if @is_shutting_down
|
52
|
-
break
|
53
|
-
end
|
54
50
|
if @poll_interval > 0 && !@is_shutting_down
|
55
51
|
Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
|
56
52
|
sleep(@poll_interval)
|
@@ -66,6 +62,8 @@ module Core
|
|
66
62
|
private
|
67
63
|
|
68
64
|
def sync_triggered?(connector_settings)
|
65
|
+
return false unless connector_registered?(connector_settings.service_type)
|
66
|
+
|
69
67
|
unless connector_settings.valid_index_name?
|
70
68
|
Utility::Logger.warn("The index name of #{connector_settings.formatted} is invalid.")
|
71
69
|
return false
|
@@ -131,6 +129,8 @@ module Core
|
|
131
129
|
end
|
132
130
|
|
133
131
|
def heartbeat_triggered?(connector_settings)
|
132
|
+
return false unless connector_registered?(connector_settings.service_type)
|
133
|
+
|
134
134
|
last_seen = connector_settings[:last_seen]
|
135
135
|
return true if last_seen.nil? || last_seen.empty?
|
136
136
|
last_seen = begin
|
@@ -144,41 +144,11 @@ module Core
|
|
144
144
|
end
|
145
145
|
|
146
146
|
def configuration_triggered?(connector_settings)
|
147
|
-
connector_settings.needs_service_type? || connector_settings.
|
148
|
-
|
149
|
-
|
150
|
-
def filtering_validation_triggered?(connector_settings)
|
151
|
-
filtering = connector_settings.filtering
|
152
|
-
|
153
|
-
unless filtering.present?
|
154
|
-
Utility::Logger.debug("#{connector_settings.formatted} does not contain filtering to be validated.")
|
155
|
-
|
156
|
-
return false
|
147
|
+
if connector_settings.needs_service_type? || connector_registered?(connector_settings.service_type)
|
148
|
+
return connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
|
157
149
|
end
|
158
150
|
|
159
|
-
|
160
|
-
|
161
|
-
unless draft_filters.present?
|
162
|
-
Utility::Logger.debug("#{connector_settings.formatted} does not contain a draft filter to be validated.")
|
163
|
-
|
164
|
-
return false
|
165
|
-
end
|
166
|
-
|
167
|
-
validation = draft_filters[:validation]
|
168
|
-
|
169
|
-
unless validation.present?
|
170
|
-
Utility::Logger.warn("#{connector_settings.formatted} does not contain a validation object inside draft filtering. Check connectors index.")
|
171
|
-
|
172
|
-
return false
|
173
|
-
end
|
174
|
-
|
175
|
-
unless validation[:state] == Core::Filtering::ValidationStatus::EDITED
|
176
|
-
Utility::Logger.debug("#{connector_settings.formatted} filtering validation needs to be in state #{Core::Filtering::ValidationStatus::EDITED} to be able to validate it.")
|
177
|
-
|
178
|
-
return false
|
179
|
-
end
|
180
|
-
|
181
|
-
true
|
151
|
+
false
|
182
152
|
end
|
183
153
|
|
184
154
|
def connector_registered?(service_type)
|
@@ -20,7 +20,7 @@ module Core
|
|
20
20
|
|
21
21
|
def connector_settings
|
22
22
|
connector_settings = Core::ConnectorSettings.fetch_by_id(@connector_id)
|
23
|
-
[connector_settings]
|
23
|
+
[connector_settings]
|
24
24
|
rescue *Utility::AUTHORIZATION_ERRORS => e
|
25
25
|
# should be handled by the general scheduler
|
26
26
|
raise e
|
data/lib/core/sync_job_runner.rb
CHANGED
@@ -8,9 +8,7 @@
|
|
8
8
|
|
9
9
|
require 'connectors/connector_status'
|
10
10
|
require 'connectors/registry'
|
11
|
-
require 'core/
|
12
|
-
require 'core/ingestion'
|
13
|
-
require 'core/filtering/validation_status'
|
11
|
+
require 'core/output_sink'
|
14
12
|
require 'utility'
|
15
13
|
|
16
14
|
module Core
|
@@ -21,21 +19,16 @@ module Core
|
|
21
19
|
end
|
22
20
|
|
23
21
|
class SyncJobRunner
|
24
|
-
|
25
|
-
|
26
|
-
def initialize(connector_settings, job)
|
22
|
+
def initialize(connector_settings)
|
27
23
|
@connector_settings = connector_settings
|
28
|
-
@sink = Core::
|
24
|
+
@sink = Core::OutputSink::EsSink.new(connector_settings.index_name, @connector_settings.request_pipeline)
|
29
25
|
@connector_class = Connectors::REGISTRY.connector_class(connector_settings.service_type)
|
30
26
|
@sync_finished = false
|
31
|
-
@sync_error = nil
|
32
27
|
@status = {
|
33
28
|
:indexed_document_count => 0,
|
34
29
|
:deleted_document_count => 0,
|
35
|
-
:indexed_document_volume => 0,
|
36
30
|
:error => nil
|
37
31
|
}
|
38
|
-
@job = job
|
39
32
|
end
|
40
33
|
|
41
34
|
def execute
|
@@ -48,17 +41,8 @@ module Core
|
|
48
41
|
def do_sync!
|
49
42
|
Utility::Logger.info("Claiming a sync job for connector #{@connector_settings.id}.")
|
50
43
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
Core::ElasticConnectorActions.update_connector_last_sync_status(@connector_settings.id, Connectors::SyncStatus::IN_PROGRESS)
|
55
|
-
|
56
|
-
# claim the job
|
57
|
-
@job.make_running!
|
58
|
-
|
59
|
-
job_description = @job.es_source
|
60
|
-
job_id = @job.id
|
61
|
-
job_description['_id'] = job_id
|
44
|
+
job_description = ElasticConnectorActions.claim_job(@connector_settings.id)
|
45
|
+
job_id = job_description['_id']
|
62
46
|
|
63
47
|
unless job_id.present?
|
64
48
|
Utility::Logger.error("Failed to claim the job for #{@connector_settings.id}. Please check the logs for the cause of this error.")
|
@@ -68,10 +52,6 @@ module Core
|
|
68
52
|
begin
|
69
53
|
Utility::Logger.debug("Successfully claimed job for connector #{@connector_settings.id}.")
|
70
54
|
|
71
|
-
Utility::Logger.info("Checking active filtering for sync job #{job_id} for connector #{@connector_settings.id}.")
|
72
|
-
validate_filtering(job_description.dig(:connector, :filtering))
|
73
|
-
Utility::Logger.debug("Active filtering for sync job #{job_id} for connector #{@connector_settings.id} is valid.")
|
74
|
-
|
75
55
|
connector_instance = Connectors::REGISTRY.connector(@connector_settings.service_type, @connector_settings.configuration, job_description: job_description)
|
76
56
|
|
77
57
|
connector_instance.do_health_check!
|
@@ -81,21 +61,11 @@ module Core
|
|
81
61
|
|
82
62
|
Utility::Logger.debug("#{existing_ids.size} documents are present in index #{@connector_settings.index_name}.")
|
83
63
|
|
84
|
-
post_processing_engine = Core::Filtering::PostProcessEngine.new(job_description)
|
85
|
-
reporting_cycle_start = Time.now
|
86
|
-
Utility::Logger.info('Yielding documents...')
|
87
64
|
connector_instance.yield_documents do |document|
|
88
65
|
document = add_ingest_metadata(document)
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
incoming_ids << document['id']
|
93
|
-
end
|
94
|
-
|
95
|
-
if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
|
96
|
-
ElasticConnectorActions.update_sync(job_id, @sink.ingestion_stats.merge(:metadata => connector_instance.metadata))
|
97
|
-
reporting_cycle_start = Time.now
|
98
|
-
end
|
66
|
+
@sink.ingest(document)
|
67
|
+
incoming_ids << document['id']
|
68
|
+
@status[:indexed_document_count] += 1
|
99
69
|
end
|
100
70
|
|
101
71
|
ids_to_delete = existing_ids - incoming_ids.uniq
|
@@ -104,11 +74,7 @@ module Core
|
|
104
74
|
|
105
75
|
ids_to_delete.each do |id|
|
106
76
|
@sink.delete(id)
|
107
|
-
|
108
|
-
if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
|
109
|
-
ElasticConnectorActions.update_sync(job_id, @sink.ingestion_stats.merge(:metadata => connector_instance.metadata))
|
110
|
-
reporting_cycle_start = Time.now
|
111
|
-
end
|
77
|
+
@status[:deleted_document_count] += 1
|
112
78
|
end
|
113
79
|
|
114
80
|
@sink.flush
|
@@ -117,34 +83,22 @@ module Core
|
|
117
83
|
# occurred as most of the time the main execution thread is interrupted and we miss this Signal/Exception here
|
118
84
|
@sync_finished = true
|
119
85
|
rescue StandardError => e
|
120
|
-
@
|
86
|
+
@status[:error] = e.message
|
121
87
|
Utility::ExceptionTracking.log_exception(e)
|
88
|
+
ElasticConnectorActions.update_connector_status(@connector_settings.id, Connectors::ConnectorStatus::ERROR, Utility::Logger.abbreviated_message(e.message))
|
122
89
|
ensure
|
123
|
-
stats = @sink.ingestion_stats
|
124
|
-
|
125
|
-
Utility::Logger.debug("Sync stats are: #{stats}")
|
126
|
-
|
127
|
-
@status[:indexed_document_count] = stats[:indexed_document_count]
|
128
|
-
@status[:deleted_document_count] = stats[:deleted_document_count]
|
129
|
-
@status[:indexed_document_volume] = stats[:indexed_document_volume]
|
130
|
-
|
131
90
|
Utility::Logger.info("Upserted #{@status[:indexed_document_count]} documents into #{@connector_settings.index_name}.")
|
132
91
|
Utility::Logger.info("Deleted #{@status[:deleted_document_count]} documents into #{@connector_settings.index_name}.")
|
133
92
|
|
134
93
|
# Make sure to not override a previous error message
|
135
|
-
if !@sync_finished && @
|
136
|
-
@
|
137
|
-
end
|
138
|
-
|
139
|
-
unless connector_instance.nil?
|
140
|
-
metadata = @sink.ingestion_stats.merge(:metadata => connector_instance.metadata)
|
141
|
-
metadata[:total_document_count] = ElasticConnectorActions.document_count(@connector_settings.index_name)
|
94
|
+
if !@sync_finished && @status[:error].nil?
|
95
|
+
@status[:error] = 'Sync thread didn\'t finish execution. Check connector logs for more details.'
|
142
96
|
end
|
143
97
|
|
144
|
-
ElasticConnectorActions.complete_sync(@connector_settings.id, job_id,
|
98
|
+
ElasticConnectorActions.complete_sync(@connector_settings.id, job_id, @status.dup)
|
145
99
|
|
146
|
-
if @
|
147
|
-
Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@
|
100
|
+
if @status[:error]
|
101
|
+
Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@status[:error]}'.")
|
148
102
|
else
|
149
103
|
Utility::Logger.info("Successfully synced for connector #{@connector_settings.id}.")
|
150
104
|
end
|
@@ -165,15 +119,5 @@ module Core
|
|
165
119
|
|
166
120
|
raise IncompatibleConfigurableFieldsError.new(@connector_class.service_type, expected_fields, actual_fields) if expected_fields != actual_fields
|
167
121
|
end
|
168
|
-
|
169
|
-
def validate_filtering(filtering)
|
170
|
-
validation_result = @connector_class.validate_filtering(filtering)
|
171
|
-
|
172
|
-
wrong_state_error = Utility::InvalidFilterConfigError.new("Active filtering is not in valid state (current state: #{validation_result[:state]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
|
173
|
-
raise wrong_state_error if validation_result[:state] != Core::Filtering::ValidationStatus::VALID
|
174
|
-
|
175
|
-
errors_present_error = Utility::InvalidFilterConfigError.new("Active filtering is in valid state, but errors were detected (errors: #{validation_result[:errors]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
|
176
|
-
raise errors_present_error if validation_result[:errors].present?
|
177
|
-
end
|
178
122
|
end
|
179
123
|
end
|
data/lib/core.rb
CHANGED
@@ -7,14 +7,10 @@
|
|
7
7
|
# frozen_string_literal: true
|
8
8
|
|
9
9
|
require 'core/configuration'
|
10
|
-
require 'core/connector_job'
|
11
10
|
require 'core/connector_settings'
|
12
11
|
require 'core/elastic_connector_actions'
|
13
|
-
require 'core/filtering'
|
14
12
|
require 'core/heartbeat'
|
15
13
|
require 'core/scheduler'
|
16
14
|
require 'core/single_scheduler'
|
17
15
|
require 'core/native_scheduler'
|
18
16
|
require 'core/sync_job_runner'
|
19
|
-
require 'core/jobs/producer'
|
20
|
-
require 'core/jobs/consumer'
|
data/lib/utility/constants.rb
CHANGED
data/lib/utility/errors.rb
CHANGED
@@ -60,6 +60,18 @@ module Utility
|
|
60
60
|
class JobDocumentLimitError < StandardError; end
|
61
61
|
class JobClaimingError < StandardError; end
|
62
62
|
|
63
|
+
class MonitoringError < StandardError
|
64
|
+
attr_accessor :tripped_by
|
65
|
+
|
66
|
+
def initialize(message = nil, tripped_by: nil)
|
67
|
+
super("#{message}#{tripped_by.present? ? " Tripped by - #{tripped_by.class}: #{tripped_by.message}" : ''}")
|
68
|
+
@tripped_by = tripped_by
|
69
|
+
end
|
70
|
+
end
|
71
|
+
class MaxSuccessiveErrorsExceededError < MonitoringError; end
|
72
|
+
class MaxErrorsExceededError < MonitoringError; end
|
73
|
+
class MaxErrorsInWindowExceededError < MonitoringError; end
|
74
|
+
|
63
75
|
class JobSyncNotPossibleYetError < StandardError
|
64
76
|
attr_accessor :sync_will_be_possible_at
|
65
77
|
|
data/lib/utility/logger.rb
CHANGED
@@ -23,7 +23,7 @@ module Utility
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def logger
|
26
|
-
@logger ||=
|
26
|
+
@logger ||= Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
|
27
27
|
end
|
28
28
|
|
29
29
|
SUPPORTED_LOG_LEVELS.each do |level|
|
data/lib/utility.rb
CHANGED
@@ -4,21 +4,14 @@
|
|
4
4
|
# you may not use this file except in compliance with the Elastic License.
|
5
5
|
#
|
6
6
|
|
7
|
-
# !!!!!!!!
|
8
|
-
# IF YOU EDIT THIS FILE, YOU MUST EDIT THE `connectors_utility.gemspec`
|
9
|
-
require 'utility/bulk_queue'
|
10
|
-
require 'utility/common'
|
11
7
|
require 'utility/constants'
|
12
8
|
require 'utility/cron'
|
13
|
-
require 'utility/
|
14
|
-
require 'utility/elasticsearch/index/text_analysis_settings'
|
15
|
-
require 'utility/environment'
|
16
|
-
require 'utility/error_monitor'
|
9
|
+
require 'utility/common'
|
17
10
|
require 'utility/errors'
|
18
|
-
require 'utility/filtering'
|
19
11
|
require 'utility/es_client'
|
12
|
+
require 'utility/environment'
|
20
13
|
require 'utility/exception_tracking'
|
21
14
|
require 'utility/extension_mapping_util'
|
22
15
|
require 'utility/logger'
|
23
|
-
|
24
|
-
|
16
|
+
require 'utility/elasticsearch/index/mappings'
|
17
|
+
require 'utility/elasticsearch/index/text_analysis_settings'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: connectors_service
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 8.6.0.4
|
4
|
+
version: 8.6.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-11-
|
11
|
+
date: 2022-11-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -396,49 +396,34 @@ files:
|
|
396
396
|
- lib/app/version.rb
|
397
397
|
- lib/connectors.rb
|
398
398
|
- lib/connectors/base/adapter.rb
|
399
|
-
- lib/connectors/base/advanced_snippet_against_schema_validator.rb
|
400
|
-
- lib/connectors/base/advanced_snippet_validator.rb
|
401
399
|
- lib/connectors/base/connector.rb
|
402
400
|
- lib/connectors/base/custom_client.rb
|
403
|
-
- lib/connectors/base/simple_rules_parser.rb
|
404
401
|
- lib/connectors/connector_status.rb
|
405
402
|
- lib/connectors/crawler/scheduler.rb
|
406
403
|
- lib/connectors/example/attachments/first_attachment.txt
|
407
404
|
- lib/connectors/example/attachments/second_attachment.txt
|
408
405
|
- lib/connectors/example/attachments/third_attachment.txt
|
409
406
|
- lib/connectors/example/connector.rb
|
410
|
-
- lib/connectors/example/example_advanced_snippet_validator.rb
|
411
407
|
- lib/connectors/gitlab/adapter.rb
|
412
408
|
- lib/connectors/gitlab/connector.rb
|
413
409
|
- lib/connectors/gitlab/custom_client.rb
|
414
410
|
- lib/connectors/gitlab/extractor.rb
|
415
|
-
- lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb
|
416
411
|
- lib/connectors/mongodb/connector.rb
|
417
|
-
- lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb
|
418
|
-
- lib/connectors/mongodb/mongo_advanced_snippet_schema.rb
|
419
|
-
- lib/connectors/mongodb/mongo_rules_parser.rb
|
420
412
|
- lib/connectors/registry.rb
|
421
413
|
- lib/connectors/sync_status.rb
|
422
|
-
- lib/connectors/tolerable_error_helper.rb
|
423
414
|
- lib/connectors_service.rb
|
424
415
|
- lib/connectors_utility.rb
|
425
416
|
- lib/core.rb
|
426
417
|
- lib/core/configuration.rb
|
427
|
-
- lib/core/connector_job.rb
|
428
418
|
- lib/core/connector_settings.rb
|
429
419
|
- lib/core/elastic_connector_actions.rb
|
430
|
-
- lib/core/filtering.rb
|
431
|
-
- lib/core/filtering/post_process_engine.rb
|
432
|
-
- lib/core/filtering/post_process_result.rb
|
433
|
-
- lib/core/filtering/simple_rule.rb
|
434
|
-
- lib/core/filtering/validation_job_runner.rb
|
435
|
-
- lib/core/filtering/validation_status.rb
|
436
420
|
- lib/core/heartbeat.rb
|
437
|
-
- lib/core/ingestion.rb
|
438
|
-
- lib/core/ingestion/es_sink.rb
|
439
|
-
- lib/core/jobs/consumer.rb
|
440
|
-
- lib/core/jobs/producer.rb
|
441
421
|
- lib/core/native_scheduler.rb
|
422
|
+
- lib/core/output_sink.rb
|
423
|
+
- lib/core/output_sink/base_sink.rb
|
424
|
+
- lib/core/output_sink/combined_sink.rb
|
425
|
+
- lib/core/output_sink/console_sink.rb
|
426
|
+
- lib/core/output_sink/es_sink.rb
|
442
427
|
- lib/core/scheduler.rb
|
443
428
|
- lib/core/single_scheduler.rb
|
444
429
|
- lib/core/sync_job_runner.rb
|
@@ -447,7 +432,6 @@ files:
|
|
447
432
|
- lib/stubs/connectors/stats.rb
|
448
433
|
- lib/stubs/service_type.rb
|
449
434
|
- lib/utility.rb
|
450
|
-
- lib/utility/bulk_queue.rb
|
451
435
|
- lib/utility/common.rb
|
452
436
|
- lib/utility/constants.rb
|
453
437
|
- lib/utility/cron.rb
|
@@ -455,12 +439,10 @@ files:
|
|
455
439
|
- lib/utility/elasticsearch/index/mappings.rb
|
456
440
|
- lib/utility/elasticsearch/index/text_analysis_settings.rb
|
457
441
|
- lib/utility/environment.rb
|
458
|
-
- lib/utility/error_monitor.rb
|
459
442
|
- lib/utility/errors.rb
|
460
443
|
- lib/utility/es_client.rb
|
461
444
|
- lib/utility/exception_tracking.rb
|
462
445
|
- lib/utility/extension_mapping_util.rb
|
463
|
-
- lib/utility/filtering.rb
|
464
446
|
- lib/utility/logger.rb
|
465
447
|
- lib/utility/middleware/basic_auth.rb
|
466
448
|
- lib/utility/middleware/bearer_auth.rb
|
@@ -480,9 +462,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
480
462
|
version: '0'
|
481
463
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
482
464
|
requirements:
|
483
|
-
- - "
|
465
|
+
- - ">="
|
484
466
|
- !ruby/object:Gem::Version
|
485
|
-
version:
|
467
|
+
version: '0'
|
486
468
|
requirements: []
|
487
469
|
rubygems_version: 3.0.3.1
|
488
470
|
signing_key:
|
@@ -1,173 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
3
|
-
# or more contributor license agreements. Licensed under the Elastic License;
|
4
|
-
# you may not use this file except in compliance with the Elastic License.
|
5
|
-
#
|
6
|
-
# frozen_string_literal: true
|
7
|
-
|
8
|
-
require 'active_support/core_ext/hash'
|
9
|
-
require 'utility/logger'
|
10
|
-
require 'connectors/base/advanced_snippet_validator'
|
11
|
-
require 'core/filtering/validation_status'
|
12
|
-
|
13
|
-
module Connectors
|
14
|
-
module Base
|
15
|
-
class AdvancedSnippetAgainstSchemaValidator < Connectors::Base::AdvancedSnippetValidator
|
16
|
-
|
17
|
-
MAX_RECURSION_DEPTH = 50
|
18
|
-
ADVANCED_SNIPPET_ID = 'advanced_snippet'
|
19
|
-
|
20
|
-
def initialize(advanced_snippet, schema)
|
21
|
-
super(advanced_snippet)
|
22
|
-
@schema = schema
|
23
|
-
end
|
24
|
-
|
25
|
-
def is_snippet_valid?
|
26
|
-
validation_result = validate_against_schema(@schema, @advanced_snippet)
|
27
|
-
log_validation_result(validation_result)
|
28
|
-
validation_result
|
29
|
-
end
|
30
|
-
|
31
|
-
private
|
32
|
-
|
33
|
-
def validate_against_schema(config_schema, advanced_snippet, recursion_depth = 0)
|
34
|
-
# Prevent unintentional/intentional SystemStackErrors/crashes
|
35
|
-
return unexpected_error if exceeded_recursion_depth?(recursion_depth)
|
36
|
-
|
37
|
-
return valid_snippet if config_schema.nil? || config_schema.empty?
|
38
|
-
|
39
|
-
schema_fields = config_schema[:fields].is_a?(Hash) ? config_schema.dig(:fields, :values) : config_schema[:fields]
|
40
|
-
snippet_field_names = advanced_snippet&.keys&.map(&:to_s)
|
41
|
-
schema_field_names = schema_fields.map { |field| field[:name] }
|
42
|
-
|
43
|
-
return unexpected_field(schema_field_names, snippet_field_names) if unexpected_field_present?(snippet_field_names, schema_field_names)
|
44
|
-
|
45
|
-
return fields_constraint_violation(config_schema[:fields]) if fields_constraints_violated?(config_schema, advanced_snippet)
|
46
|
-
|
47
|
-
schema_fields.each do |field|
|
48
|
-
name = field[:name]
|
49
|
-
type = field[:type]
|
50
|
-
optional = field[:optional] || false
|
51
|
-
|
52
|
-
snippet_field_value = advanced_snippet.with_indifferent_access[name]
|
53
|
-
|
54
|
-
next if optional && (snippet_field_value.nil? || !snippet_field_value.present?)
|
55
|
-
|
56
|
-
return wrong_names(snippet_field_names, name) unless snippet_field_names.include?(name)
|
57
|
-
|
58
|
-
return wrong_type(name, type, snippet_field_value) if type_error_present?(type, snippet_field_value)
|
59
|
-
|
60
|
-
if field[:fields].present?
|
61
|
-
validation_result = validate_against_schema(field, snippet_field_value, recursion_depth + 1)
|
62
|
-
|
63
|
-
return validation_result unless validation_result[:is_valid]
|
64
|
-
end
|
65
|
-
end
|
66
|
-
|
67
|
-
valid_snippet
|
68
|
-
end
|
69
|
-
|
70
|
-
def fields_constraints_violated?(config_schema, advanced_snippet)
|
71
|
-
return false unless config_schema[:fields].is_a?(Hash)
|
72
|
-
|
73
|
-
constraints = config_schema.dig(:fields, :constraints)
|
74
|
-
constraints = constraints.is_a?(Array) ? constraints : [constraints]
|
75
|
-
|
76
|
-
constraints.each do |constraint|
|
77
|
-
return true unless constraint.call(advanced_snippet)
|
78
|
-
end
|
79
|
-
|
80
|
-
false
|
81
|
-
end
|
82
|
-
|
83
|
-
def type_error_present?(schema_type, snippet_value)
|
84
|
-
return !schema_type.call(snippet_value) if schema_type.is_a?(Proc)
|
85
|
-
|
86
|
-
!snippet_value.is_a?(schema_type)
|
87
|
-
end
|
88
|
-
|
89
|
-
def exceeded_recursion_depth?(recursion_depth)
|
90
|
-
if recursion_depth >= MAX_RECURSION_DEPTH
|
91
|
-
Utility::Logger.warn("Recursion depth for filtering validation exceeded. (Max recursion depth: #{MAX_RECURSION_DEPTH})")
|
92
|
-
return true
|
93
|
-
end
|
94
|
-
|
95
|
-
false
|
96
|
-
end
|
97
|
-
|
98
|
-
def unexpected_field_present?(actual_field_names, expected_field_names)
|
99
|
-
difference = actual_field_names - expected_field_names
|
100
|
-
|
101
|
-
# we have field names, which we didn't expect
|
102
|
-
!difference.empty?
|
103
|
-
end
|
104
|
-
|
105
|
-
def valid_snippet
|
106
|
-
{
|
107
|
-
:state => Core::Filtering::ValidationStatus::VALID,
|
108
|
-
:errors => []
|
109
|
-
}
|
110
|
-
end
|
111
|
-
|
112
|
-
def unexpected_field(expected_fields, actual_fields)
|
113
|
-
{
|
114
|
-
:state => Core::Filtering::ValidationStatus::INVALID,
|
115
|
-
:errors => [
|
116
|
-
{
|
117
|
-
:ids => [ADVANCED_SNIPPET_ID],
|
118
|
-
:messages => ["Encountered unexpected fields '#{actual_fields}'. Expected: '#{expected_fields}'."]
|
119
|
-
}
|
120
|
-
]
|
121
|
-
}
|
122
|
-
end
|
123
|
-
|
124
|
-
def wrong_type(field_name, expected_type, actual_value)
|
125
|
-
{
|
126
|
-
:state => Core::Filtering::ValidationStatus::INVALID,
|
127
|
-
:errors => [
|
128
|
-
{
|
129
|
-
:ids => [ADVANCED_SNIPPET_ID],
|
130
|
-
:messages => ["Expected field type '#{expected_type.is_a?(Proc) ? 'custom matcher' : expected_type}' for field '#{field_name}', but got value '#{actual_value.inspect}' of type '#{actual_value.class}'."]
|
131
|
-
}
|
132
|
-
]
|
133
|
-
}
|
134
|
-
end
|
135
|
-
|
136
|
-
def wrong_names(actual_field_names, expected_field_name)
|
137
|
-
{
|
138
|
-
:state => Core::Filtering::ValidationStatus::INVALID,
|
139
|
-
:errors => [
|
140
|
-
{
|
141
|
-
:ids => [ADVANCED_SNIPPET_ID],
|
142
|
-
:messages => ["Expected field name '#{expected_field_name}', but got #{actual_field_names}."]
|
143
|
-
}
|
144
|
-
]
|
145
|
-
}
|
146
|
-
end
|
147
|
-
|
148
|
-
def fields_constraint_violation(fields)
|
149
|
-
{
|
150
|
-
:state => Core::Filtering::ValidationStatus::INVALID,
|
151
|
-
:errors => [
|
152
|
-
{
|
153
|
-
:ids => [ADVANCED_SNIPPET_ID],
|
154
|
-
:messages => ["A fields constraint was violated for fields: '#{fields[:values].map { |v| v[:name] }}'. Check advanced snippet field constraints."]
|
155
|
-
}
|
156
|
-
]
|
157
|
-
}
|
158
|
-
end
|
159
|
-
|
160
|
-
def unexpected_error
|
161
|
-
{
|
162
|
-
:state => Core::Filtering::ValidationStatus::INVALID,
|
163
|
-
:errors => [
|
164
|
-
{
|
165
|
-
:ids => [ADVANCED_SNIPPET_ID],
|
166
|
-
:messages => ['Unexpected error. Check logs for details.']
|
167
|
-
}
|
168
|
-
]
|
169
|
-
}
|
170
|
-
end
|
171
|
-
end
|
172
|
-
end
|
173
|
-
end
|
@@ -1,34 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
3
|
-
# or more contributor license agreements. Licensed under the Elastic License;
|
4
|
-
# you may not use this file except in compliance with the Elastic License.
|
5
|
-
#
|
6
|
-
# frozen_string_literal: true
|
7
|
-
|
8
|
-
require 'utility/logger'
|
9
|
-
|
10
|
-
module Connectors
|
11
|
-
module Base
|
12
|
-
class AdvancedSnippetValidator
|
13
|
-
|
14
|
-
def initialize(advanced_snippet)
|
15
|
-
@advanced_snippet = advanced_snippet || {}
|
16
|
-
end
|
17
|
-
|
18
|
-
def is_snippet_valid?
|
19
|
-
raise 'Advanced Snippet validation not implemented'
|
20
|
-
end
|
21
|
-
|
22
|
-
private
|
23
|
-
|
24
|
-
def log_validation_result(validation_result)
|
25
|
-
Utility::Logger.info("Filtering Advanced Configuration validation result: #{validation_result[:state]}")
|
26
|
-
if validation_result[:errors].present?
|
27
|
-
validation_result[:errors].each do |error|
|
28
|
-
Utility::Logger.warn("Validation error for: '#{error[:ids]}': '#{error[:messages]}'")
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|