connectors_service 8.6.0.4.pre.20221116T024501Z → 8.6.0.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/config/connectors.yml +6 -6
  3. data/lib/app/app.rb +0 -4
  4. data/lib/app/dispatcher.rb +17 -42
  5. data/lib/app/preflight_check.rb +0 -11
  6. data/lib/connectors/base/connector.rb +14 -43
  7. data/lib/connectors/example/connector.rb +0 -6
  8. data/lib/connectors/gitlab/connector.rb +1 -6
  9. data/lib/connectors/mongodb/connector.rb +43 -47
  10. data/lib/connectors/sync_status.rb +1 -6
  11. data/lib/core/configuration.rb +1 -3
  12. data/lib/core/connector_settings.rb +16 -52
  13. data/lib/core/elastic_connector_actions.rb +59 -320
  14. data/lib/core/output_sink/base_sink.rb +33 -0
  15. data/lib/core/output_sink/combined_sink.rb +38 -0
  16. data/lib/core/output_sink/console_sink.rb +51 -0
  17. data/lib/core/output_sink/es_sink.rb +74 -0
  18. data/lib/core/{ingestion.rb → output_sink.rb} +5 -1
  19. data/lib/core/scheduler.rb +10 -40
  20. data/lib/core/single_scheduler.rb +1 -1
  21. data/lib/core/sync_job_runner.rb +16 -72
  22. data/lib/core.rb +0 -4
  23. data/lib/utility/constants.rb +0 -2
  24. data/lib/utility/errors.rb +12 -0
  25. data/lib/utility/logger.rb +1 -1
  26. data/lib/utility.rb +4 -11
  27. metadata +9 -27
  28. data/lib/connectors/base/advanced_snippet_against_schema_validator.rb +0 -173
  29. data/lib/connectors/base/advanced_snippet_validator.rb +0 -34
  30. data/lib/connectors/base/simple_rules_parser.rb +0 -42
  31. data/lib/connectors/example/example_advanced_snippet_validator.rb +0 -35
  32. data/lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb +0 -35
  33. data/lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb +0 -22
  34. data/lib/connectors/mongodb/mongo_advanced_snippet_schema.rb +0 -292
  35. data/lib/connectors/mongodb/mongo_rules_parser.rb +0 -81
  36. data/lib/connectors/tolerable_error_helper.rb +0 -43
  37. data/lib/core/connector_job.rb +0 -210
  38. data/lib/core/filtering/post_process_engine.rb +0 -39
  39. data/lib/core/filtering/post_process_result.rb +0 -27
  40. data/lib/core/filtering/simple_rule.rb +0 -141
  41. data/lib/core/filtering/validation_job_runner.rb +0 -53
  42. data/lib/core/filtering/validation_status.rb +0 -17
  43. data/lib/core/filtering.rb +0 -17
  44. data/lib/core/ingestion/es_sink.rb +0 -118
  45. data/lib/core/jobs/consumer.rb +0 -114
  46. data/lib/core/jobs/producer.rb +0 -26
  47. data/lib/utility/bulk_queue.rb +0 -85
  48. data/lib/utility/error_monitor.rb +0 -108
  49. data/lib/utility/filtering.rb +0 -22
@@ -6,4 +6,8 @@
6
6
 
7
7
  # frozen_string_literal: true
8
8
 
9
- require 'core/ingestion/es_sink'
9
+ require 'core/output_sink/es_sink'
10
+ require 'core/output_sink/console_sink'
11
+ require 'core/output_sink/combined_sink'
12
+
13
+ module Core::OutputSink; end
@@ -10,7 +10,6 @@ require 'time'
10
10
  require 'fugit'
11
11
  require 'core/connector_settings'
12
12
  require 'core/elastic_connector_actions'
13
- require 'core/filtering/validation_status'
14
13
  require 'utility/cron'
15
14
  require 'utility/logger'
16
15
  require 'utility/exception_tracking'
@@ -39,18 +38,15 @@ module Core
39
38
  if configuration_triggered?(cs)
40
39
  yield cs, :configuration
41
40
  end
42
- if filtering_validation_triggered?(cs)
43
- yield cs, :filter_validation
44
- end
41
+ end
42
+ if @is_shutting_down
43
+ break
45
44
  end
46
45
  rescue *Utility::AUTHORIZATION_ERRORS => e
47
46
  Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
48
47
  rescue StandardError => e
49
48
  Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
50
49
  ensure
51
- if @is_shutting_down
52
- break
53
- end
54
50
  if @poll_interval > 0 && !@is_shutting_down
55
51
  Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
56
52
  sleep(@poll_interval)
@@ -66,6 +62,8 @@ module Core
66
62
  private
67
63
 
68
64
  def sync_triggered?(connector_settings)
65
+ return false unless connector_registered?(connector_settings.service_type)
66
+
69
67
  unless connector_settings.valid_index_name?
70
68
  Utility::Logger.warn("The index name of #{connector_settings.formatted} is invalid.")
71
69
  return false
@@ -131,6 +129,8 @@ module Core
131
129
  end
132
130
 
133
131
  def heartbeat_triggered?(connector_settings)
132
+ return false unless connector_registered?(connector_settings.service_type)
133
+
134
134
  last_seen = connector_settings[:last_seen]
135
135
  return true if last_seen.nil? || last_seen.empty?
136
136
  last_seen = begin
@@ -144,41 +144,11 @@ module Core
144
144
  end
145
145
 
146
146
  def configuration_triggered?(connector_settings)
147
- connector_settings.needs_service_type? || connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
148
- end
149
-
150
- def filtering_validation_triggered?(connector_settings)
151
- filtering = connector_settings.filtering
152
-
153
- unless filtering.present?
154
- Utility::Logger.debug("#{connector_settings.formatted} does not contain filtering to be validated.")
155
-
156
- return false
147
+ if connector_settings.needs_service_type? || connector_registered?(connector_settings.service_type)
148
+ return connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
157
149
  end
158
150
 
159
- draft_filters = filtering[:draft]
160
-
161
- unless draft_filters.present?
162
- Utility::Logger.debug("#{connector_settings.formatted} does not contain a draft filter to be validated.")
163
-
164
- return false
165
- end
166
-
167
- validation = draft_filters[:validation]
168
-
169
- unless validation.present?
170
- Utility::Logger.warn("#{connector_settings.formatted} does not contain a validation object inside draft filtering. Check connectors index.")
171
-
172
- return false
173
- end
174
-
175
- unless validation[:state] == Core::Filtering::ValidationStatus::EDITED
176
- Utility::Logger.debug("#{connector_settings.formatted} filtering validation needs to be in state #{Core::Filtering::ValidationStatus::EDITED} to be able to validate it.")
177
-
178
- return false
179
- end
180
-
181
- true
151
+ false
182
152
  end
183
153
 
184
154
  def connector_registered?(service_type)
@@ -20,7 +20,7 @@ module Core
20
20
 
21
21
  def connector_settings
22
22
  connector_settings = Core::ConnectorSettings.fetch_by_id(@connector_id)
23
- [connector_settings].compact
23
+ [connector_settings]
24
24
  rescue *Utility::AUTHORIZATION_ERRORS => e
25
25
  # should be handled by the general scheduler
26
26
  raise e
@@ -8,9 +8,7 @@
8
8
 
9
9
  require 'connectors/connector_status'
10
10
  require 'connectors/registry'
11
- require 'core/filtering/post_process_engine'
12
- require 'core/ingestion'
13
- require 'core/filtering/validation_status'
11
+ require 'core/output_sink'
14
12
  require 'utility'
15
13
 
16
14
  module Core
@@ -21,21 +19,16 @@ module Core
21
19
  end
22
20
 
23
21
  class SyncJobRunner
24
- JOB_REPORTING_INTERVAL = 10
25
-
26
- def initialize(connector_settings, job)
22
+ def initialize(connector_settings)
27
23
  @connector_settings = connector_settings
28
- @sink = Core::Ingestion::EsSink.new(connector_settings.index_name, @connector_settings.request_pipeline)
24
+ @sink = Core::OutputSink::EsSink.new(connector_settings.index_name, @connector_settings.request_pipeline)
29
25
  @connector_class = Connectors::REGISTRY.connector_class(connector_settings.service_type)
30
26
  @sync_finished = false
31
- @sync_error = nil
32
27
  @status = {
33
28
  :indexed_document_count => 0,
34
29
  :deleted_document_count => 0,
35
- :indexed_document_volume => 0,
36
30
  :error => nil
37
31
  }
38
- @job = job
39
32
  end
40
33
 
41
34
  def execute
@@ -48,17 +41,8 @@ module Core
48
41
  def do_sync!
49
42
  Utility::Logger.info("Claiming a sync job for connector #{@connector_settings.id}.")
50
43
 
51
- # connector service doesn't support multiple jobs running simultaneously
52
- raise Core::JobAlreadyRunningError.new(@connector_settings.id) if @connector_settings.running?
53
-
54
- Core::ElasticConnectorActions.update_connector_last_sync_status(@connector_settings.id, Connectors::SyncStatus::IN_PROGRESS)
55
-
56
- # claim the job
57
- @job.make_running!
58
-
59
- job_description = @job.es_source
60
- job_id = @job.id
61
- job_description['_id'] = job_id
44
+ job_description = ElasticConnectorActions.claim_job(@connector_settings.id)
45
+ job_id = job_description['_id']
62
46
 
63
47
  unless job_id.present?
64
48
  Utility::Logger.error("Failed to claim the job for #{@connector_settings.id}. Please check the logs for the cause of this error.")
@@ -68,10 +52,6 @@ module Core
68
52
  begin
69
53
  Utility::Logger.debug("Successfully claimed job for connector #{@connector_settings.id}.")
70
54
 
71
- Utility::Logger.info("Checking active filtering for sync job #{job_id} for connector #{@connector_settings.id}.")
72
- validate_filtering(job_description.dig(:connector, :filtering))
73
- Utility::Logger.debug("Active filtering for sync job #{job_id} for connector #{@connector_settings.id} is valid.")
74
-
75
55
  connector_instance = Connectors::REGISTRY.connector(@connector_settings.service_type, @connector_settings.configuration, job_description: job_description)
76
56
 
77
57
  connector_instance.do_health_check!
@@ -81,21 +61,11 @@ module Core
81
61
 
82
62
  Utility::Logger.debug("#{existing_ids.size} documents are present in index #{@connector_settings.index_name}.")
83
63
 
84
- post_processing_engine = Core::Filtering::PostProcessEngine.new(job_description)
85
- reporting_cycle_start = Time.now
86
- Utility::Logger.info('Yielding documents...')
87
64
  connector_instance.yield_documents do |document|
88
65
  document = add_ingest_metadata(document)
89
- post_process_result = post_processing_engine.process(document)
90
- if post_process_result.is_include?
91
- @sink.ingest(document)
92
- incoming_ids << document['id']
93
- end
94
-
95
- if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
96
- ElasticConnectorActions.update_sync(job_id, @sink.ingestion_stats.merge(:metadata => connector_instance.metadata))
97
- reporting_cycle_start = Time.now
98
- end
66
+ @sink.ingest(document)
67
+ incoming_ids << document['id']
68
+ @status[:indexed_document_count] += 1
99
69
  end
100
70
 
101
71
  ids_to_delete = existing_ids - incoming_ids.uniq
@@ -104,11 +74,7 @@ module Core
104
74
 
105
75
  ids_to_delete.each do |id|
106
76
  @sink.delete(id)
107
-
108
- if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
109
- ElasticConnectorActions.update_sync(job_id, @sink.ingestion_stats.merge(:metadata => connector_instance.metadata))
110
- reporting_cycle_start = Time.now
111
- end
77
+ @status[:deleted_document_count] += 1
112
78
  end
113
79
 
114
80
  @sink.flush
@@ -117,34 +83,22 @@ module Core
117
83
  # occurred as most of the time the main execution thread is interrupted and we miss this Signal/Exception here
118
84
  @sync_finished = true
119
85
  rescue StandardError => e
120
- @sync_error = e.message
86
+ @status[:error] = e.message
121
87
  Utility::ExceptionTracking.log_exception(e)
88
+ ElasticConnectorActions.update_connector_status(@connector_settings.id, Connectors::ConnectorStatus::ERROR, Utility::Logger.abbreviated_message(e.message))
122
89
  ensure
123
- stats = @sink.ingestion_stats
124
-
125
- Utility::Logger.debug("Sync stats are: #{stats}")
126
-
127
- @status[:indexed_document_count] = stats[:indexed_document_count]
128
- @status[:deleted_document_count] = stats[:deleted_document_count]
129
- @status[:indexed_document_volume] = stats[:indexed_document_volume]
130
-
131
90
  Utility::Logger.info("Upserted #{@status[:indexed_document_count]} documents into #{@connector_settings.index_name}.")
132
91
  Utility::Logger.info("Deleted #{@status[:deleted_document_count]} documents into #{@connector_settings.index_name}.")
133
92
 
134
93
  # Make sure to not override a previous error message
135
- if !@sync_finished && @sync_error.nil?
136
- @sync_error = 'Sync thread didn\'t finish execution. Check connector logs for more details.'
137
- end
138
-
139
- unless connector_instance.nil?
140
- metadata = @sink.ingestion_stats.merge(:metadata => connector_instance.metadata)
141
- metadata[:total_document_count] = ElasticConnectorActions.document_count(@connector_settings.index_name)
94
+ if !@sync_finished && @status[:error].nil?
95
+ @status[:error] = 'Sync thread didn\'t finish execution. Check connector logs for more details.'
142
96
  end
143
97
 
144
- ElasticConnectorActions.complete_sync(@connector_settings.id, job_id, metadata, @sync_error)
98
+ ElasticConnectorActions.complete_sync(@connector_settings.id, job_id, @status.dup)
145
99
 
146
- if @sync_error
147
- Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@sync_error}'.")
100
+ if @status[:error]
101
+ Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@status[:error]}'.")
148
102
  else
149
103
  Utility::Logger.info("Successfully synced for connector #{@connector_settings.id}.")
150
104
  end
@@ -165,15 +119,5 @@ module Core
165
119
 
166
120
  raise IncompatibleConfigurableFieldsError.new(@connector_class.service_type, expected_fields, actual_fields) if expected_fields != actual_fields
167
121
  end
168
-
169
- def validate_filtering(filtering)
170
- validation_result = @connector_class.validate_filtering(filtering)
171
-
172
- wrong_state_error = Utility::InvalidFilterConfigError.new("Active filtering is not in valid state (current state: #{validation_result[:state]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
173
- raise wrong_state_error if validation_result[:state] != Core::Filtering::ValidationStatus::VALID
174
-
175
- errors_present_error = Utility::InvalidFilterConfigError.new("Active filtering is in valid state, but errors were detected (errors: #{validation_result[:errors]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
176
- raise errors_present_error if validation_result[:errors].present?
177
- end
178
122
  end
179
123
  end
data/lib/core.rb CHANGED
@@ -7,14 +7,10 @@
7
7
  # frozen_string_literal: true
8
8
 
9
9
  require 'core/configuration'
10
- require 'core/connector_job'
11
10
  require 'core/connector_settings'
12
11
  require 'core/elastic_connector_actions'
13
- require 'core/filtering'
14
12
  require 'core/heartbeat'
15
13
  require 'core/scheduler'
16
14
  require 'core/single_scheduler'
17
15
  require 'core/native_scheduler'
18
16
  require 'core/sync_job_runner'
19
- require 'core/jobs/producer'
20
- require 'core/jobs/consumer'
@@ -16,7 +16,5 @@ module Utility
16
16
  JOB_INDEX = '.elastic-connectors-sync-jobs'
17
17
  CONTENT_INDEX_PREFIX = 'search-'
18
18
  CRAWLER_SERVICE_TYPE = 'elastic-crawler'
19
- FILTERING_RULES_FEATURE = 'filtering_rules'
20
- FILTERING_ADVANCED_FEATURE = 'filtering_advanced_config'
21
19
  end
22
20
  end
@@ -60,6 +60,18 @@ module Utility
60
60
  class JobDocumentLimitError < StandardError; end
61
61
  class JobClaimingError < StandardError; end
62
62
 
63
+ class MonitoringError < StandardError
64
+ attr_accessor :tripped_by
65
+
66
+ def initialize(message = nil, tripped_by: nil)
67
+ super("#{message}#{tripped_by.present? ? " Tripped by - #{tripped_by.class}: #{tripped_by.message}" : ''}")
68
+ @tripped_by = tripped_by
69
+ end
70
+ end
71
+ class MaxSuccessiveErrorsExceededError < MonitoringError; end
72
+ class MaxErrorsExceededError < MonitoringError; end
73
+ class MaxErrorsInWindowExceededError < MonitoringError; end
74
+
63
75
  class JobSyncNotPossibleYetError < StandardError
64
76
  attr_accessor :sync_will_be_possible_at
65
77
 
@@ -23,7 +23,7 @@ module Utility
23
23
  end
24
24
 
25
25
  def logger
26
- @logger ||= defined?(::Settings) && ::Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
26
+ @logger ||= Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
27
27
  end
28
28
 
29
29
  SUPPORTED_LOG_LEVELS.each do |level|
data/lib/utility.rb CHANGED
@@ -4,21 +4,14 @@
4
4
  # you may not use this file except in compliance with the Elastic License.
5
5
  #
6
6
 
7
- # !!!!!!!!
8
- # IF YOU EDIT THIS FILE, YOU MUST EDIT THE `connectors_utility.gemspec`
9
- require 'utility/bulk_queue'
10
- require 'utility/common'
11
7
  require 'utility/constants'
12
8
  require 'utility/cron'
13
- require 'utility/elasticsearch/index/mappings'
14
- require 'utility/elasticsearch/index/text_analysis_settings'
15
- require 'utility/environment'
16
- require 'utility/error_monitor'
9
+ require 'utility/common'
17
10
  require 'utility/errors'
18
- require 'utility/filtering'
19
11
  require 'utility/es_client'
12
+ require 'utility/environment'
20
13
  require 'utility/exception_tracking'
21
14
  require 'utility/extension_mapping_util'
22
15
  require 'utility/logger'
23
- # IF YOU EDIT THIS FILE, YOU MUST EDIT THE `connectors_utility.gemspec`
24
- # !!!!!!!!
16
+ require 'utility/elasticsearch/index/mappings'
17
+ require 'utility/elasticsearch/index/text_analysis_settings'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: connectors_service
3
3
  version: !ruby/object:Gem::Version
4
- version: 8.6.0.4.pre.20221116T024501Z
4
+ version: 8.6.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-11-16 00:00:00.000000000 Z
11
+ date: 2022-11-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -396,49 +396,34 @@ files:
396
396
  - lib/app/version.rb
397
397
  - lib/connectors.rb
398
398
  - lib/connectors/base/adapter.rb
399
- - lib/connectors/base/advanced_snippet_against_schema_validator.rb
400
- - lib/connectors/base/advanced_snippet_validator.rb
401
399
  - lib/connectors/base/connector.rb
402
400
  - lib/connectors/base/custom_client.rb
403
- - lib/connectors/base/simple_rules_parser.rb
404
401
  - lib/connectors/connector_status.rb
405
402
  - lib/connectors/crawler/scheduler.rb
406
403
  - lib/connectors/example/attachments/first_attachment.txt
407
404
  - lib/connectors/example/attachments/second_attachment.txt
408
405
  - lib/connectors/example/attachments/third_attachment.txt
409
406
  - lib/connectors/example/connector.rb
410
- - lib/connectors/example/example_advanced_snippet_validator.rb
411
407
  - lib/connectors/gitlab/adapter.rb
412
408
  - lib/connectors/gitlab/connector.rb
413
409
  - lib/connectors/gitlab/custom_client.rb
414
410
  - lib/connectors/gitlab/extractor.rb
415
- - lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb
416
411
  - lib/connectors/mongodb/connector.rb
417
- - lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb
418
- - lib/connectors/mongodb/mongo_advanced_snippet_schema.rb
419
- - lib/connectors/mongodb/mongo_rules_parser.rb
420
412
  - lib/connectors/registry.rb
421
413
  - lib/connectors/sync_status.rb
422
- - lib/connectors/tolerable_error_helper.rb
423
414
  - lib/connectors_service.rb
424
415
  - lib/connectors_utility.rb
425
416
  - lib/core.rb
426
417
  - lib/core/configuration.rb
427
- - lib/core/connector_job.rb
428
418
  - lib/core/connector_settings.rb
429
419
  - lib/core/elastic_connector_actions.rb
430
- - lib/core/filtering.rb
431
- - lib/core/filtering/post_process_engine.rb
432
- - lib/core/filtering/post_process_result.rb
433
- - lib/core/filtering/simple_rule.rb
434
- - lib/core/filtering/validation_job_runner.rb
435
- - lib/core/filtering/validation_status.rb
436
420
  - lib/core/heartbeat.rb
437
- - lib/core/ingestion.rb
438
- - lib/core/ingestion/es_sink.rb
439
- - lib/core/jobs/consumer.rb
440
- - lib/core/jobs/producer.rb
441
421
  - lib/core/native_scheduler.rb
422
+ - lib/core/output_sink.rb
423
+ - lib/core/output_sink/base_sink.rb
424
+ - lib/core/output_sink/combined_sink.rb
425
+ - lib/core/output_sink/console_sink.rb
426
+ - lib/core/output_sink/es_sink.rb
442
427
  - lib/core/scheduler.rb
443
428
  - lib/core/single_scheduler.rb
444
429
  - lib/core/sync_job_runner.rb
@@ -447,7 +432,6 @@ files:
447
432
  - lib/stubs/connectors/stats.rb
448
433
  - lib/stubs/service_type.rb
449
434
  - lib/utility.rb
450
- - lib/utility/bulk_queue.rb
451
435
  - lib/utility/common.rb
452
436
  - lib/utility/constants.rb
453
437
  - lib/utility/cron.rb
@@ -455,12 +439,10 @@ files:
455
439
  - lib/utility/elasticsearch/index/mappings.rb
456
440
  - lib/utility/elasticsearch/index/text_analysis_settings.rb
457
441
  - lib/utility/environment.rb
458
- - lib/utility/error_monitor.rb
459
442
  - lib/utility/errors.rb
460
443
  - lib/utility/es_client.rb
461
444
  - lib/utility/exception_tracking.rb
462
445
  - lib/utility/extension_mapping_util.rb
463
- - lib/utility/filtering.rb
464
446
  - lib/utility/logger.rb
465
447
  - lib/utility/middleware/basic_auth.rb
466
448
  - lib/utility/middleware/bearer_auth.rb
@@ -480,9 +462,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
480
462
  version: '0'
481
463
  required_rubygems_version: !ruby/object:Gem::Requirement
482
464
  requirements:
483
- - - ">"
465
+ - - ">="
484
466
  - !ruby/object:Gem::Version
485
- version: 1.3.1
467
+ version: '0'
486
468
  requirements: []
487
469
  rubygems_version: 3.0.3.1
488
470
  signing_key:
@@ -1,173 +0,0 @@
1
- #
2
- # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3
- # or more contributor license agreements. Licensed under the Elastic License;
4
- # you may not use this file except in compliance with the Elastic License.
5
- #
6
- # frozen_string_literal: true
7
-
8
- require 'active_support/core_ext/hash'
9
- require 'utility/logger'
10
- require 'connectors/base/advanced_snippet_validator'
11
- require 'core/filtering/validation_status'
12
-
13
- module Connectors
14
- module Base
15
- class AdvancedSnippetAgainstSchemaValidator < Connectors::Base::AdvancedSnippetValidator
16
-
17
- MAX_RECURSION_DEPTH = 50
18
- ADVANCED_SNIPPET_ID = 'advanced_snippet'
19
-
20
- def initialize(advanced_snippet, schema)
21
- super(advanced_snippet)
22
- @schema = schema
23
- end
24
-
25
- def is_snippet_valid?
26
- validation_result = validate_against_schema(@schema, @advanced_snippet)
27
- log_validation_result(validation_result)
28
- validation_result
29
- end
30
-
31
- private
32
-
33
- def validate_against_schema(config_schema, advanced_snippet, recursion_depth = 0)
34
- # Prevent unintentional/intentional SystemStackErrors/crashes
35
- return unexpected_error if exceeded_recursion_depth?(recursion_depth)
36
-
37
- return valid_snippet if config_schema.nil? || config_schema.empty?
38
-
39
- schema_fields = config_schema[:fields].is_a?(Hash) ? config_schema.dig(:fields, :values) : config_schema[:fields]
40
- snippet_field_names = advanced_snippet&.keys&.map(&:to_s)
41
- schema_field_names = schema_fields.map { |field| field[:name] }
42
-
43
- return unexpected_field(schema_field_names, snippet_field_names) if unexpected_field_present?(snippet_field_names, schema_field_names)
44
-
45
- return fields_constraint_violation(config_schema[:fields]) if fields_constraints_violated?(config_schema, advanced_snippet)
46
-
47
- schema_fields.each do |field|
48
- name = field[:name]
49
- type = field[:type]
50
- optional = field[:optional] || false
51
-
52
- snippet_field_value = advanced_snippet.with_indifferent_access[name]
53
-
54
- next if optional && (snippet_field_value.nil? || !snippet_field_value.present?)
55
-
56
- return wrong_names(snippet_field_names, name) unless snippet_field_names.include?(name)
57
-
58
- return wrong_type(name, type, snippet_field_value) if type_error_present?(type, snippet_field_value)
59
-
60
- if field[:fields].present?
61
- validation_result = validate_against_schema(field, snippet_field_value, recursion_depth + 1)
62
-
63
- return validation_result unless validation_result[:is_valid]
64
- end
65
- end
66
-
67
- valid_snippet
68
- end
69
-
70
- def fields_constraints_violated?(config_schema, advanced_snippet)
71
- return false unless config_schema[:fields].is_a?(Hash)
72
-
73
- constraints = config_schema.dig(:fields, :constraints)
74
- constraints = constraints.is_a?(Array) ? constraints : [constraints]
75
-
76
- constraints.each do |constraint|
77
- return true unless constraint.call(advanced_snippet)
78
- end
79
-
80
- false
81
- end
82
-
83
- def type_error_present?(schema_type, snippet_value)
84
- return !schema_type.call(snippet_value) if schema_type.is_a?(Proc)
85
-
86
- !snippet_value.is_a?(schema_type)
87
- end
88
-
89
- def exceeded_recursion_depth?(recursion_depth)
90
- if recursion_depth >= MAX_RECURSION_DEPTH
91
- Utility::Logger.warn("Recursion depth for filtering validation exceeded. (Max recursion depth: #{MAX_RECURSION_DEPTH})")
92
- return true
93
- end
94
-
95
- false
96
- end
97
-
98
- def unexpected_field_present?(actual_field_names, expected_field_names)
99
- difference = actual_field_names - expected_field_names
100
-
101
- # we have field names, which we didn't expect
102
- !difference.empty?
103
- end
104
-
105
- def valid_snippet
106
- {
107
- :state => Core::Filtering::ValidationStatus::VALID,
108
- :errors => []
109
- }
110
- end
111
-
112
- def unexpected_field(expected_fields, actual_fields)
113
- {
114
- :state => Core::Filtering::ValidationStatus::INVALID,
115
- :errors => [
116
- {
117
- :ids => [ADVANCED_SNIPPET_ID],
118
- :messages => ["Encountered unexpected fields '#{actual_fields}'. Expected: '#{expected_fields}'."]
119
- }
120
- ]
121
- }
122
- end
123
-
124
- def wrong_type(field_name, expected_type, actual_value)
125
- {
126
- :state => Core::Filtering::ValidationStatus::INVALID,
127
- :errors => [
128
- {
129
- :ids => [ADVANCED_SNIPPET_ID],
130
- :messages => ["Expected field type '#{expected_type.is_a?(Proc) ? 'custom matcher' : expected_type}' for field '#{field_name}', but got value '#{actual_value.inspect}' of type '#{actual_value.class}'."]
131
- }
132
- ]
133
- }
134
- end
135
-
136
- def wrong_names(actual_field_names, expected_field_name)
137
- {
138
- :state => Core::Filtering::ValidationStatus::INVALID,
139
- :errors => [
140
- {
141
- :ids => [ADVANCED_SNIPPET_ID],
142
- :messages => ["Expected field name '#{expected_field_name}', but got #{actual_field_names}."]
143
- }
144
- ]
145
- }
146
- end
147
-
148
- def fields_constraint_violation(fields)
149
- {
150
- :state => Core::Filtering::ValidationStatus::INVALID,
151
- :errors => [
152
- {
153
- :ids => [ADVANCED_SNIPPET_ID],
154
- :messages => ["A fields constraint was violated for fields: '#{fields[:values].map { |v| v[:name] }}'. Check advanced snippet field constraints."]
155
- }
156
- ]
157
- }
158
- end
159
-
160
- def unexpected_error
161
- {
162
- :state => Core::Filtering::ValidationStatus::INVALID,
163
- :errors => [
164
- {
165
- :ids => [ADVANCED_SNIPPET_ID],
166
- :messages => ['Unexpected error. Check logs for details.']
167
- }
168
- ]
169
- }
170
- end
171
- end
172
- end
173
- end
@@ -1,34 +0,0 @@
1
- #
2
- # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3
- # or more contributor license agreements. Licensed under the Elastic License;
4
- # you may not use this file except in compliance with the Elastic License.
5
- #
6
- # frozen_string_literal: true
7
-
8
- require 'utility/logger'
9
-
10
- module Connectors
11
- module Base
12
- class AdvancedSnippetValidator
13
-
14
- def initialize(advanced_snippet)
15
- @advanced_snippet = advanced_snippet || {}
16
- end
17
-
18
- def is_snippet_valid?
19
- raise 'Advanced Snippet validation not implemented'
20
- end
21
-
22
- private
23
-
24
- def log_validation_result(validation_result)
25
- Utility::Logger.info("Filtering Advanced Configuration validation result: #{validation_result[:state]}")
26
- if validation_result[:errors].present?
27
- validation_result[:errors].each do |error|
28
- Utility::Logger.warn("Validation error for: '#{error[:ids]}': '#{error[:messages]}'")
29
- end
30
- end
31
- end
32
- end
33
- end
34
- end