connectors_service 8.6.0.4.pre.20221116T024501Z → 8.6.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/config/connectors.yml +6 -6
  3. data/lib/app/app.rb +0 -4
  4. data/lib/app/dispatcher.rb +17 -42
  5. data/lib/app/preflight_check.rb +0 -11
  6. data/lib/connectors/base/connector.rb +14 -43
  7. data/lib/connectors/example/connector.rb +0 -6
  8. data/lib/connectors/gitlab/connector.rb +1 -6
  9. data/lib/connectors/mongodb/connector.rb +43 -47
  10. data/lib/connectors/sync_status.rb +1 -6
  11. data/lib/core/configuration.rb +1 -3
  12. data/lib/core/connector_settings.rb +16 -52
  13. data/lib/core/elastic_connector_actions.rb +59 -320
  14. data/lib/core/output_sink/base_sink.rb +33 -0
  15. data/lib/core/output_sink/combined_sink.rb +38 -0
  16. data/lib/core/output_sink/console_sink.rb +51 -0
  17. data/lib/core/output_sink/es_sink.rb +74 -0
  18. data/lib/core/{ingestion.rb → output_sink.rb} +5 -1
  19. data/lib/core/scheduler.rb +10 -40
  20. data/lib/core/single_scheduler.rb +1 -1
  21. data/lib/core/sync_job_runner.rb +16 -72
  22. data/lib/core.rb +0 -4
  23. data/lib/utility/constants.rb +0 -2
  24. data/lib/utility/errors.rb +12 -0
  25. data/lib/utility/logger.rb +1 -1
  26. data/lib/utility.rb +4 -11
  27. metadata +9 -27
  28. data/lib/connectors/base/advanced_snippet_against_schema_validator.rb +0 -173
  29. data/lib/connectors/base/advanced_snippet_validator.rb +0 -34
  30. data/lib/connectors/base/simple_rules_parser.rb +0 -42
  31. data/lib/connectors/example/example_advanced_snippet_validator.rb +0 -35
  32. data/lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb +0 -35
  33. data/lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb +0 -22
  34. data/lib/connectors/mongodb/mongo_advanced_snippet_schema.rb +0 -292
  35. data/lib/connectors/mongodb/mongo_rules_parser.rb +0 -81
  36. data/lib/connectors/tolerable_error_helper.rb +0 -43
  37. data/lib/core/connector_job.rb +0 -210
  38. data/lib/core/filtering/post_process_engine.rb +0 -39
  39. data/lib/core/filtering/post_process_result.rb +0 -27
  40. data/lib/core/filtering/simple_rule.rb +0 -141
  41. data/lib/core/filtering/validation_job_runner.rb +0 -53
  42. data/lib/core/filtering/validation_status.rb +0 -17
  43. data/lib/core/filtering.rb +0 -17
  44. data/lib/core/ingestion/es_sink.rb +0 -118
  45. data/lib/core/jobs/consumer.rb +0 -114
  46. data/lib/core/jobs/producer.rb +0 -26
  47. data/lib/utility/bulk_queue.rb +0 -85
  48. data/lib/utility/error_monitor.rb +0 -108
  49. data/lib/utility/filtering.rb +0 -22
@@ -6,4 +6,8 @@
6
6
 
7
7
  # frozen_string_literal: true
8
8
 
9
- require 'core/ingestion/es_sink'
9
+ require 'core/output_sink/es_sink'
10
+ require 'core/output_sink/console_sink'
11
+ require 'core/output_sink/combined_sink'
12
+
13
+ module Core::OutputSink; end
@@ -10,7 +10,6 @@ require 'time'
10
10
  require 'fugit'
11
11
  require 'core/connector_settings'
12
12
  require 'core/elastic_connector_actions'
13
- require 'core/filtering/validation_status'
14
13
  require 'utility/cron'
15
14
  require 'utility/logger'
16
15
  require 'utility/exception_tracking'
@@ -39,18 +38,15 @@ module Core
39
38
  if configuration_triggered?(cs)
40
39
  yield cs, :configuration
41
40
  end
42
- if filtering_validation_triggered?(cs)
43
- yield cs, :filter_validation
44
- end
41
+ end
42
+ if @is_shutting_down
43
+ break
45
44
  end
46
45
  rescue *Utility::AUTHORIZATION_ERRORS => e
47
46
  Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
48
47
  rescue StandardError => e
49
48
  Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
50
49
  ensure
51
- if @is_shutting_down
52
- break
53
- end
54
50
  if @poll_interval > 0 && !@is_shutting_down
55
51
  Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
56
52
  sleep(@poll_interval)
@@ -66,6 +62,8 @@ module Core
66
62
  private
67
63
 
68
64
  def sync_triggered?(connector_settings)
65
+ return false unless connector_registered?(connector_settings.service_type)
66
+
69
67
  unless connector_settings.valid_index_name?
70
68
  Utility::Logger.warn("The index name of #{connector_settings.formatted} is invalid.")
71
69
  return false
@@ -131,6 +129,8 @@ module Core
131
129
  end
132
130
 
133
131
  def heartbeat_triggered?(connector_settings)
132
+ return false unless connector_registered?(connector_settings.service_type)
133
+
134
134
  last_seen = connector_settings[:last_seen]
135
135
  return true if last_seen.nil? || last_seen.empty?
136
136
  last_seen = begin
@@ -144,41 +144,11 @@ module Core
144
144
  end
145
145
 
146
146
  def configuration_triggered?(connector_settings)
147
- connector_settings.needs_service_type? || connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
148
- end
149
-
150
- def filtering_validation_triggered?(connector_settings)
151
- filtering = connector_settings.filtering
152
-
153
- unless filtering.present?
154
- Utility::Logger.debug("#{connector_settings.formatted} does not contain filtering to be validated.")
155
-
156
- return false
147
+ if connector_settings.needs_service_type? || connector_registered?(connector_settings.service_type)
148
+ return connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
157
149
  end
158
150
 
159
- draft_filters = filtering[:draft]
160
-
161
- unless draft_filters.present?
162
- Utility::Logger.debug("#{connector_settings.formatted} does not contain a draft filter to be validated.")
163
-
164
- return false
165
- end
166
-
167
- validation = draft_filters[:validation]
168
-
169
- unless validation.present?
170
- Utility::Logger.warn("#{connector_settings.formatted} does not contain a validation object inside draft filtering. Check connectors index.")
171
-
172
- return false
173
- end
174
-
175
- unless validation[:state] == Core::Filtering::ValidationStatus::EDITED
176
- Utility::Logger.debug("#{connector_settings.formatted} filtering validation needs to be in state #{Core::Filtering::ValidationStatus::EDITED} to be able to validate it.")
177
-
178
- return false
179
- end
180
-
181
- true
151
+ false
182
152
  end
183
153
 
184
154
  def connector_registered?(service_type)
@@ -20,7 +20,7 @@ module Core
20
20
 
21
21
  def connector_settings
22
22
  connector_settings = Core::ConnectorSettings.fetch_by_id(@connector_id)
23
- [connector_settings].compact
23
+ [connector_settings]
24
24
  rescue *Utility::AUTHORIZATION_ERRORS => e
25
25
  # should be handled by the general scheduler
26
26
  raise e
@@ -8,9 +8,7 @@
8
8
 
9
9
  require 'connectors/connector_status'
10
10
  require 'connectors/registry'
11
- require 'core/filtering/post_process_engine'
12
- require 'core/ingestion'
13
- require 'core/filtering/validation_status'
11
+ require 'core/output_sink'
14
12
  require 'utility'
15
13
 
16
14
  module Core
@@ -21,21 +19,16 @@ module Core
21
19
  end
22
20
 
23
21
  class SyncJobRunner
24
- JOB_REPORTING_INTERVAL = 10
25
-
26
- def initialize(connector_settings, job)
22
+ def initialize(connector_settings)
27
23
  @connector_settings = connector_settings
28
- @sink = Core::Ingestion::EsSink.new(connector_settings.index_name, @connector_settings.request_pipeline)
24
+ @sink = Core::OutputSink::EsSink.new(connector_settings.index_name, @connector_settings.request_pipeline)
29
25
  @connector_class = Connectors::REGISTRY.connector_class(connector_settings.service_type)
30
26
  @sync_finished = false
31
- @sync_error = nil
32
27
  @status = {
33
28
  :indexed_document_count => 0,
34
29
  :deleted_document_count => 0,
35
- :indexed_document_volume => 0,
36
30
  :error => nil
37
31
  }
38
- @job = job
39
32
  end
40
33
 
41
34
  def execute
@@ -48,17 +41,8 @@ module Core
48
41
  def do_sync!
49
42
  Utility::Logger.info("Claiming a sync job for connector #{@connector_settings.id}.")
50
43
 
51
- # connector service doesn't support multiple jobs running simultaneously
52
- raise Core::JobAlreadyRunningError.new(@connector_settings.id) if @connector_settings.running?
53
-
54
- Core::ElasticConnectorActions.update_connector_last_sync_status(@connector_settings.id, Connectors::SyncStatus::IN_PROGRESS)
55
-
56
- # claim the job
57
- @job.make_running!
58
-
59
- job_description = @job.es_source
60
- job_id = @job.id
61
- job_description['_id'] = job_id
44
+ job_description = ElasticConnectorActions.claim_job(@connector_settings.id)
45
+ job_id = job_description['_id']
62
46
 
63
47
  unless job_id.present?
64
48
  Utility::Logger.error("Failed to claim the job for #{@connector_settings.id}. Please check the logs for the cause of this error.")
@@ -68,10 +52,6 @@ module Core
68
52
  begin
69
53
  Utility::Logger.debug("Successfully claimed job for connector #{@connector_settings.id}.")
70
54
 
71
- Utility::Logger.info("Checking active filtering for sync job #{job_id} for connector #{@connector_settings.id}.")
72
- validate_filtering(job_description.dig(:connector, :filtering))
73
- Utility::Logger.debug("Active filtering for sync job #{job_id} for connector #{@connector_settings.id} is valid.")
74
-
75
55
  connector_instance = Connectors::REGISTRY.connector(@connector_settings.service_type, @connector_settings.configuration, job_description: job_description)
76
56
 
77
57
  connector_instance.do_health_check!
@@ -81,21 +61,11 @@ module Core
81
61
 
82
62
  Utility::Logger.debug("#{existing_ids.size} documents are present in index #{@connector_settings.index_name}.")
83
63
 
84
- post_processing_engine = Core::Filtering::PostProcessEngine.new(job_description)
85
- reporting_cycle_start = Time.now
86
- Utility::Logger.info('Yielding documents...')
87
64
  connector_instance.yield_documents do |document|
88
65
  document = add_ingest_metadata(document)
89
- post_process_result = post_processing_engine.process(document)
90
- if post_process_result.is_include?
91
- @sink.ingest(document)
92
- incoming_ids << document['id']
93
- end
94
-
95
- if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
96
- ElasticConnectorActions.update_sync(job_id, @sink.ingestion_stats.merge(:metadata => connector_instance.metadata))
97
- reporting_cycle_start = Time.now
98
- end
66
+ @sink.ingest(document)
67
+ incoming_ids << document['id']
68
+ @status[:indexed_document_count] += 1
99
69
  end
100
70
 
101
71
  ids_to_delete = existing_ids - incoming_ids.uniq
@@ -104,11 +74,7 @@ module Core
104
74
 
105
75
  ids_to_delete.each do |id|
106
76
  @sink.delete(id)
107
-
108
- if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
109
- ElasticConnectorActions.update_sync(job_id, @sink.ingestion_stats.merge(:metadata => connector_instance.metadata))
110
- reporting_cycle_start = Time.now
111
- end
77
+ @status[:deleted_document_count] += 1
112
78
  end
113
79
 
114
80
  @sink.flush
@@ -117,34 +83,22 @@ module Core
117
83
  # occurred as most of the time the main execution thread is interrupted and we miss this Signal/Exception here
118
84
  @sync_finished = true
119
85
  rescue StandardError => e
120
- @sync_error = e.message
86
+ @status[:error] = e.message
121
87
  Utility::ExceptionTracking.log_exception(e)
88
+ ElasticConnectorActions.update_connector_status(@connector_settings.id, Connectors::ConnectorStatus::ERROR, Utility::Logger.abbreviated_message(e.message))
122
89
  ensure
123
- stats = @sink.ingestion_stats
124
-
125
- Utility::Logger.debug("Sync stats are: #{stats}")
126
-
127
- @status[:indexed_document_count] = stats[:indexed_document_count]
128
- @status[:deleted_document_count] = stats[:deleted_document_count]
129
- @status[:indexed_document_volume] = stats[:indexed_document_volume]
130
-
131
90
  Utility::Logger.info("Upserted #{@status[:indexed_document_count]} documents into #{@connector_settings.index_name}.")
132
91
  Utility::Logger.info("Deleted #{@status[:deleted_document_count]} documents into #{@connector_settings.index_name}.")
133
92
 
134
93
  # Make sure to not override a previous error message
135
- if !@sync_finished && @sync_error.nil?
136
- @sync_error = 'Sync thread didn\'t finish execution. Check connector logs for more details.'
137
- end
138
-
139
- unless connector_instance.nil?
140
- metadata = @sink.ingestion_stats.merge(:metadata => connector_instance.metadata)
141
- metadata[:total_document_count] = ElasticConnectorActions.document_count(@connector_settings.index_name)
94
+ if !@sync_finished && @status[:error].nil?
95
+ @status[:error] = 'Sync thread didn\'t finish execution. Check connector logs for more details.'
142
96
  end
143
97
 
144
- ElasticConnectorActions.complete_sync(@connector_settings.id, job_id, metadata, @sync_error)
98
+ ElasticConnectorActions.complete_sync(@connector_settings.id, job_id, @status.dup)
145
99
 
146
- if @sync_error
147
- Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@sync_error}'.")
100
+ if @status[:error]
101
+ Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@status[:error]}'.")
148
102
  else
149
103
  Utility::Logger.info("Successfully synced for connector #{@connector_settings.id}.")
150
104
  end
@@ -165,15 +119,5 @@ module Core
165
119
 
166
120
  raise IncompatibleConfigurableFieldsError.new(@connector_class.service_type, expected_fields, actual_fields) if expected_fields != actual_fields
167
121
  end
168
-
169
- def validate_filtering(filtering)
170
- validation_result = @connector_class.validate_filtering(filtering)
171
-
172
- wrong_state_error = Utility::InvalidFilterConfigError.new("Active filtering is not in valid state (current state: #{validation_result[:state]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
173
- raise wrong_state_error if validation_result[:state] != Core::Filtering::ValidationStatus::VALID
174
-
175
- errors_present_error = Utility::InvalidFilterConfigError.new("Active filtering is in valid state, but errors were detected (errors: #{validation_result[:errors]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
176
- raise errors_present_error if validation_result[:errors].present?
177
- end
178
122
  end
179
123
  end
data/lib/core.rb CHANGED
@@ -7,14 +7,10 @@
7
7
  # frozen_string_literal: true
8
8
 
9
9
  require 'core/configuration'
10
- require 'core/connector_job'
11
10
  require 'core/connector_settings'
12
11
  require 'core/elastic_connector_actions'
13
- require 'core/filtering'
14
12
  require 'core/heartbeat'
15
13
  require 'core/scheduler'
16
14
  require 'core/single_scheduler'
17
15
  require 'core/native_scheduler'
18
16
  require 'core/sync_job_runner'
19
- require 'core/jobs/producer'
20
- require 'core/jobs/consumer'
@@ -16,7 +16,5 @@ module Utility
16
16
  JOB_INDEX = '.elastic-connectors-sync-jobs'
17
17
  CONTENT_INDEX_PREFIX = 'search-'
18
18
  CRAWLER_SERVICE_TYPE = 'elastic-crawler'
19
- FILTERING_RULES_FEATURE = 'filtering_rules'
20
- FILTERING_ADVANCED_FEATURE = 'filtering_advanced_config'
21
19
  end
22
20
  end
@@ -60,6 +60,18 @@ module Utility
60
60
  class JobDocumentLimitError < StandardError; end
61
61
  class JobClaimingError < StandardError; end
62
62
 
63
+ class MonitoringError < StandardError
64
+ attr_accessor :tripped_by
65
+
66
+ def initialize(message = nil, tripped_by: nil)
67
+ super("#{message}#{tripped_by.present? ? " Tripped by - #{tripped_by.class}: #{tripped_by.message}" : ''}")
68
+ @tripped_by = tripped_by
69
+ end
70
+ end
71
+ class MaxSuccessiveErrorsExceededError < MonitoringError; end
72
+ class MaxErrorsExceededError < MonitoringError; end
73
+ class MaxErrorsInWindowExceededError < MonitoringError; end
74
+
63
75
  class JobSyncNotPossibleYetError < StandardError
64
76
  attr_accessor :sync_will_be_possible_at
65
77
 
@@ -23,7 +23,7 @@ module Utility
23
23
  end
24
24
 
25
25
  def logger
26
- @logger ||= defined?(::Settings) && ::Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
26
+ @logger ||= Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
27
27
  end
28
28
 
29
29
  SUPPORTED_LOG_LEVELS.each do |level|
data/lib/utility.rb CHANGED
@@ -4,21 +4,14 @@
4
4
  # you may not use this file except in compliance with the Elastic License.
5
5
  #
6
6
 
7
- # !!!!!!!!
8
- # IF YOU EDIT THIS FILE, YOU MUST EDIT THE `connectors_utility.gemspec`
9
- require 'utility/bulk_queue'
10
- require 'utility/common'
11
7
  require 'utility/constants'
12
8
  require 'utility/cron'
13
- require 'utility/elasticsearch/index/mappings'
14
- require 'utility/elasticsearch/index/text_analysis_settings'
15
- require 'utility/environment'
16
- require 'utility/error_monitor'
9
+ require 'utility/common'
17
10
  require 'utility/errors'
18
- require 'utility/filtering'
19
11
  require 'utility/es_client'
12
+ require 'utility/environment'
20
13
  require 'utility/exception_tracking'
21
14
  require 'utility/extension_mapping_util'
22
15
  require 'utility/logger'
23
- # IF YOU EDIT THIS FILE, YOU MUST EDIT THE `connectors_utility.gemspec`
24
- # !!!!!!!!
16
+ require 'utility/elasticsearch/index/mappings'
17
+ require 'utility/elasticsearch/index/text_analysis_settings'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: connectors_service
3
3
  version: !ruby/object:Gem::Version
4
- version: 8.6.0.4.pre.20221116T024501Z
4
+ version: 8.6.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-11-16 00:00:00.000000000 Z
11
+ date: 2022-11-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -396,49 +396,34 @@ files:
396
396
  - lib/app/version.rb
397
397
  - lib/connectors.rb
398
398
  - lib/connectors/base/adapter.rb
399
- - lib/connectors/base/advanced_snippet_against_schema_validator.rb
400
- - lib/connectors/base/advanced_snippet_validator.rb
401
399
  - lib/connectors/base/connector.rb
402
400
  - lib/connectors/base/custom_client.rb
403
- - lib/connectors/base/simple_rules_parser.rb
404
401
  - lib/connectors/connector_status.rb
405
402
  - lib/connectors/crawler/scheduler.rb
406
403
  - lib/connectors/example/attachments/first_attachment.txt
407
404
  - lib/connectors/example/attachments/second_attachment.txt
408
405
  - lib/connectors/example/attachments/third_attachment.txt
409
406
  - lib/connectors/example/connector.rb
410
- - lib/connectors/example/example_advanced_snippet_validator.rb
411
407
  - lib/connectors/gitlab/adapter.rb
412
408
  - lib/connectors/gitlab/connector.rb
413
409
  - lib/connectors/gitlab/custom_client.rb
414
410
  - lib/connectors/gitlab/extractor.rb
415
- - lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb
416
411
  - lib/connectors/mongodb/connector.rb
417
- - lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb
418
- - lib/connectors/mongodb/mongo_advanced_snippet_schema.rb
419
- - lib/connectors/mongodb/mongo_rules_parser.rb
420
412
  - lib/connectors/registry.rb
421
413
  - lib/connectors/sync_status.rb
422
- - lib/connectors/tolerable_error_helper.rb
423
414
  - lib/connectors_service.rb
424
415
  - lib/connectors_utility.rb
425
416
  - lib/core.rb
426
417
  - lib/core/configuration.rb
427
- - lib/core/connector_job.rb
428
418
  - lib/core/connector_settings.rb
429
419
  - lib/core/elastic_connector_actions.rb
430
- - lib/core/filtering.rb
431
- - lib/core/filtering/post_process_engine.rb
432
- - lib/core/filtering/post_process_result.rb
433
- - lib/core/filtering/simple_rule.rb
434
- - lib/core/filtering/validation_job_runner.rb
435
- - lib/core/filtering/validation_status.rb
436
420
  - lib/core/heartbeat.rb
437
- - lib/core/ingestion.rb
438
- - lib/core/ingestion/es_sink.rb
439
- - lib/core/jobs/consumer.rb
440
- - lib/core/jobs/producer.rb
441
421
  - lib/core/native_scheduler.rb
422
+ - lib/core/output_sink.rb
423
+ - lib/core/output_sink/base_sink.rb
424
+ - lib/core/output_sink/combined_sink.rb
425
+ - lib/core/output_sink/console_sink.rb
426
+ - lib/core/output_sink/es_sink.rb
442
427
  - lib/core/scheduler.rb
443
428
  - lib/core/single_scheduler.rb
444
429
  - lib/core/sync_job_runner.rb
@@ -447,7 +432,6 @@ files:
447
432
  - lib/stubs/connectors/stats.rb
448
433
  - lib/stubs/service_type.rb
449
434
  - lib/utility.rb
450
- - lib/utility/bulk_queue.rb
451
435
  - lib/utility/common.rb
452
436
  - lib/utility/constants.rb
453
437
  - lib/utility/cron.rb
@@ -455,12 +439,10 @@ files:
455
439
  - lib/utility/elasticsearch/index/mappings.rb
456
440
  - lib/utility/elasticsearch/index/text_analysis_settings.rb
457
441
  - lib/utility/environment.rb
458
- - lib/utility/error_monitor.rb
459
442
  - lib/utility/errors.rb
460
443
  - lib/utility/es_client.rb
461
444
  - lib/utility/exception_tracking.rb
462
445
  - lib/utility/extension_mapping_util.rb
463
- - lib/utility/filtering.rb
464
446
  - lib/utility/logger.rb
465
447
  - lib/utility/middleware/basic_auth.rb
466
448
  - lib/utility/middleware/bearer_auth.rb
@@ -480,9 +462,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
480
462
  version: '0'
481
463
  required_rubygems_version: !ruby/object:Gem::Requirement
482
464
  requirements:
483
- - - ">"
465
+ - - ">="
484
466
  - !ruby/object:Gem::Version
485
- version: 1.3.1
467
+ version: '0'
486
468
  requirements: []
487
469
  rubygems_version: 3.0.3.1
488
470
  signing_key:
@@ -1,173 +0,0 @@
1
- #
2
- # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3
- # or more contributor license agreements. Licensed under the Elastic License;
4
- # you may not use this file except in compliance with the Elastic License.
5
- #
6
- # frozen_string_literal: true
7
-
8
- require 'active_support/core_ext/hash'
9
- require 'utility/logger'
10
- require 'connectors/base/advanced_snippet_validator'
11
- require 'core/filtering/validation_status'
12
-
13
- module Connectors
14
- module Base
15
- class AdvancedSnippetAgainstSchemaValidator < Connectors::Base::AdvancedSnippetValidator
16
-
17
- MAX_RECURSION_DEPTH = 50
18
- ADVANCED_SNIPPET_ID = 'advanced_snippet'
19
-
20
- def initialize(advanced_snippet, schema)
21
- super(advanced_snippet)
22
- @schema = schema
23
- end
24
-
25
- def is_snippet_valid?
26
- validation_result = validate_against_schema(@schema, @advanced_snippet)
27
- log_validation_result(validation_result)
28
- validation_result
29
- end
30
-
31
- private
32
-
33
- def validate_against_schema(config_schema, advanced_snippet, recursion_depth = 0)
34
- # Prevent unintentional/intentional SystemStackErrors/crashes
35
- return unexpected_error if exceeded_recursion_depth?(recursion_depth)
36
-
37
- return valid_snippet if config_schema.nil? || config_schema.empty?
38
-
39
- schema_fields = config_schema[:fields].is_a?(Hash) ? config_schema.dig(:fields, :values) : config_schema[:fields]
40
- snippet_field_names = advanced_snippet&.keys&.map(&:to_s)
41
- schema_field_names = schema_fields.map { |field| field[:name] }
42
-
43
- return unexpected_field(schema_field_names, snippet_field_names) if unexpected_field_present?(snippet_field_names, schema_field_names)
44
-
45
- return fields_constraint_violation(config_schema[:fields]) if fields_constraints_violated?(config_schema, advanced_snippet)
46
-
47
- schema_fields.each do |field|
48
- name = field[:name]
49
- type = field[:type]
50
- optional = field[:optional] || false
51
-
52
- snippet_field_value = advanced_snippet.with_indifferent_access[name]
53
-
54
- next if optional && (snippet_field_value.nil? || !snippet_field_value.present?)
55
-
56
- return wrong_names(snippet_field_names, name) unless snippet_field_names.include?(name)
57
-
58
- return wrong_type(name, type, snippet_field_value) if type_error_present?(type, snippet_field_value)
59
-
60
- if field[:fields].present?
61
- validation_result = validate_against_schema(field, snippet_field_value, recursion_depth + 1)
62
-
63
- return validation_result unless validation_result[:is_valid]
64
- end
65
- end
66
-
67
- valid_snippet
68
- end
69
-
70
- def fields_constraints_violated?(config_schema, advanced_snippet)
71
- return false unless config_schema[:fields].is_a?(Hash)
72
-
73
- constraints = config_schema.dig(:fields, :constraints)
74
- constraints = constraints.is_a?(Array) ? constraints : [constraints]
75
-
76
- constraints.each do |constraint|
77
- return true unless constraint.call(advanced_snippet)
78
- end
79
-
80
- false
81
- end
82
-
83
- def type_error_present?(schema_type, snippet_value)
84
- return !schema_type.call(snippet_value) if schema_type.is_a?(Proc)
85
-
86
- !snippet_value.is_a?(schema_type)
87
- end
88
-
89
- def exceeded_recursion_depth?(recursion_depth)
90
- if recursion_depth >= MAX_RECURSION_DEPTH
91
- Utility::Logger.warn("Recursion depth for filtering validation exceeded. (Max recursion depth: #{MAX_RECURSION_DEPTH})")
92
- return true
93
- end
94
-
95
- false
96
- end
97
-
98
- def unexpected_field_present?(actual_field_names, expected_field_names)
99
- difference = actual_field_names - expected_field_names
100
-
101
- # we have field names, which we didn't expect
102
- !difference.empty?
103
- end
104
-
105
- def valid_snippet
106
- {
107
- :state => Core::Filtering::ValidationStatus::VALID,
108
- :errors => []
109
- }
110
- end
111
-
112
- def unexpected_field(expected_fields, actual_fields)
113
- {
114
- :state => Core::Filtering::ValidationStatus::INVALID,
115
- :errors => [
116
- {
117
- :ids => [ADVANCED_SNIPPET_ID],
118
- :messages => ["Encountered unexpected fields '#{actual_fields}'. Expected: '#{expected_fields}'."]
119
- }
120
- ]
121
- }
122
- end
123
-
124
- def wrong_type(field_name, expected_type, actual_value)
125
- {
126
- :state => Core::Filtering::ValidationStatus::INVALID,
127
- :errors => [
128
- {
129
- :ids => [ADVANCED_SNIPPET_ID],
130
- :messages => ["Expected field type '#{expected_type.is_a?(Proc) ? 'custom matcher' : expected_type}' for field '#{field_name}', but got value '#{actual_value.inspect}' of type '#{actual_value.class}'."]
131
- }
132
- ]
133
- }
134
- end
135
-
136
- def wrong_names(actual_field_names, expected_field_name)
137
- {
138
- :state => Core::Filtering::ValidationStatus::INVALID,
139
- :errors => [
140
- {
141
- :ids => [ADVANCED_SNIPPET_ID],
142
- :messages => ["Expected field name '#{expected_field_name}', but got #{actual_field_names}."]
143
- }
144
- ]
145
- }
146
- end
147
-
148
- def fields_constraint_violation(fields)
149
- {
150
- :state => Core::Filtering::ValidationStatus::INVALID,
151
- :errors => [
152
- {
153
- :ids => [ADVANCED_SNIPPET_ID],
154
- :messages => ["A fields constraint was violated for fields: '#{fields[:values].map { |v| v[:name] }}'. Check advanced snippet field constraints."]
155
- }
156
- ]
157
- }
158
- end
159
-
160
- def unexpected_error
161
- {
162
- :state => Core::Filtering::ValidationStatus::INVALID,
163
- :errors => [
164
- {
165
- :ids => [ADVANCED_SNIPPET_ID],
166
- :messages => ['Unexpected error. Check logs for details.']
167
- }
168
- ]
169
- }
170
- end
171
- end
172
- end
173
- end
@@ -1,34 +0,0 @@
1
- #
2
- # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3
- # or more contributor license agreements. Licensed under the Elastic License;
4
- # you may not use this file except in compliance with the Elastic License.
5
- #
6
- # frozen_string_literal: true
7
-
8
- require 'utility/logger'
9
-
10
- module Connectors
11
- module Base
12
- class AdvancedSnippetValidator
13
-
14
- def initialize(advanced_snippet)
15
- @advanced_snippet = advanced_snippet || {}
16
- end
17
-
18
- def is_snippet_valid?
19
- raise 'Advanced Snippet validation not implemented'
20
- end
21
-
22
- private
23
-
24
- def log_validation_result(validation_result)
25
- Utility::Logger.info("Filtering Advanced Configuration validation result: #{validation_result[:state]}")
26
- if validation_result[:errors].present?
27
- validation_result[:errors].each do |error|
28
- Utility::Logger.warn("Validation error for: '#{error[:ids]}': '#{error[:messages]}'")
29
- end
30
- end
31
- end
32
- end
33
- end
34
- end