connectors_service 8.6.0.4.pre.20221116T024501Z → 8.6.0.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/config/connectors.yml +6 -6
  3. data/lib/app/app.rb +0 -4
  4. data/lib/app/dispatcher.rb +17 -42
  5. data/lib/app/preflight_check.rb +0 -11
  6. data/lib/connectors/base/connector.rb +14 -43
  7. data/lib/connectors/example/connector.rb +0 -6
  8. data/lib/connectors/gitlab/connector.rb +1 -6
  9. data/lib/connectors/mongodb/connector.rb +43 -47
  10. data/lib/connectors/sync_status.rb +1 -6
  11. data/lib/core/configuration.rb +1 -3
  12. data/lib/core/connector_settings.rb +16 -52
  13. data/lib/core/elastic_connector_actions.rb +59 -320
  14. data/lib/core/output_sink/base_sink.rb +33 -0
  15. data/lib/core/output_sink/combined_sink.rb +38 -0
  16. data/lib/core/output_sink/console_sink.rb +51 -0
  17. data/lib/core/output_sink/es_sink.rb +74 -0
  18. data/lib/core/{ingestion.rb → output_sink.rb} +5 -1
  19. data/lib/core/scheduler.rb +10 -40
  20. data/lib/core/single_scheduler.rb +1 -1
  21. data/lib/core/sync_job_runner.rb +16 -72
  22. data/lib/core.rb +0 -4
  23. data/lib/utility/constants.rb +0 -2
  24. data/lib/utility/errors.rb +12 -0
  25. data/lib/utility/logger.rb +1 -1
  26. data/lib/utility.rb +4 -11
  27. metadata +9 -27
  28. data/lib/connectors/base/advanced_snippet_against_schema_validator.rb +0 -173
  29. data/lib/connectors/base/advanced_snippet_validator.rb +0 -34
  30. data/lib/connectors/base/simple_rules_parser.rb +0 -42
  31. data/lib/connectors/example/example_advanced_snippet_validator.rb +0 -35
  32. data/lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb +0 -35
  33. data/lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb +0 -22
  34. data/lib/connectors/mongodb/mongo_advanced_snippet_schema.rb +0 -292
  35. data/lib/connectors/mongodb/mongo_rules_parser.rb +0 -81
  36. data/lib/connectors/tolerable_error_helper.rb +0 -43
  37. data/lib/core/connector_job.rb +0 -210
  38. data/lib/core/filtering/post_process_engine.rb +0 -39
  39. data/lib/core/filtering/post_process_result.rb +0 -27
  40. data/lib/core/filtering/simple_rule.rb +0 -141
  41. data/lib/core/filtering/validation_job_runner.rb +0 -53
  42. data/lib/core/filtering/validation_status.rb +0 -17
  43. data/lib/core/filtering.rb +0 -17
  44. data/lib/core/ingestion/es_sink.rb +0 -118
  45. data/lib/core/jobs/consumer.rb +0 -114
  46. data/lib/core/jobs/producer.rb +0 -26
  47. data/lib/utility/bulk_queue.rb +0 -85
  48. data/lib/utility/error_monitor.rb +0 -108
  49. data/lib/utility/filtering.rb +0 -22
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9a610999d9c34543392e1dd9b7b9ab136557992cc1503fa5aa5973186d17af9e
4
- data.tar.gz: '086282a4a15ce1b168e6a0add5f78646e1fe89c336b8a257677826cce3677635'
3
+ metadata.gz: 43cb60284b3e1c74a6f34612c0f7509cb4be0375d5871c6b59d7338495c88236
4
+ data.tar.gz: 3f176c8770fc9b450a244b5f8cc7a5857668a452c17e19a07ddb412dcb8d2452
5
5
  SHA512:
6
- metadata.gz: bf0a0a22ea96074bcbb904000ec489a591fc8491bc2d91759bc1a097c60e31c2abad7b118bb97d16bbe80c5b00dbe08f7c65653710c2d6a77d3356607d87e045
7
- data.tar.gz: 67ff773ed2ae3f869897b5fb3a2c0dc3fc55c5922c602e032ed5bafa800043bd4b7f4af90fdd055e575ea877f3cf3b9e308d91c58bad59cd6c63dd5eb8ab98f4
6
+ metadata.gz: 59746d5ae600b0a77aaede033f5b371cd922e3a4e95f61afda146f5b6e0962026c02c987fc43c203c5274381539f5c76e5ef895fce593ef1883fa6cf620a1702
7
+ data.tar.gz: ee799a965b0f5b028c2a41003069d3b7b1e49cabe38453629af8a382ed7ac92b76fe835986c7e5157c1a06b1c2ebae84c20606319bbe7b0aad9a771b12a3f977
@@ -1,10 +1,10 @@
1
1
  # general metadata
2
- version: 8.6.0.4-20221116T024501Z
2
+ version: 8.6.0.4-20221104T200420Z
3
3
  repository: git@github.com:elastic/ent-search-connectors.git
4
- revision: b3cc1332879a38930a272a63f8c6be1847578204
4
+ revision: 2051b3907639a1fe2ae68efdc33c06cf12d38383
5
5
  elasticsearch:
6
6
  hosts: http://localhost:9200
7
- api_key: MDJyaGZZUUJ1ZXdOLTB1ZnQ5aXQ6a2IwQURWSjVUc1NPcjVsOFBuVDBzZw==
7
+ api_key: OW1FalJJUUI1clBtUVh5RVo1QmU6QVp5LV9pU3RRUXFYb2VVYnlCRWNZdw==
8
8
  retry_on_failure: 3
9
9
  request_timeout: 120
10
10
  disable_warnings: true
@@ -14,11 +14,11 @@ thread_pool:
14
14
  min_threads: 0
15
15
  max_threads: 5
16
16
  max_queue: 100
17
- log_level: debug
17
+ log_level: info
18
18
  ecs_logging: false
19
19
  poll_interval: 3
20
20
  termination_timeout: 60
21
21
  heartbeat_interval: 1800
22
22
  native_mode: false
23
- connector_id: 0mrhfYQBuewN-0ufptgN
24
- service_type: mongodb
23
+ connector_id: 9WEjRIQB5rPmQXyEWJB2
24
+ service_type: example
data/lib/app/app.rb CHANGED
@@ -17,10 +17,6 @@ require 'utility/logger'
17
17
  module App
18
18
  Utility::Environment.set_execution_environment(App::Config) do
19
19
  App::PreflightCheck.run!
20
-
21
- # set exit hook
22
- Kernel.at_exit { App::Dispatcher.shutdown! }
23
-
24
20
  App::Dispatcher.start!
25
21
  rescue App::PreflightCheck::CheckFailure => e
26
22
  Utility::Logger.error("Preflight check failed: #{e.message}")
@@ -28,10 +28,6 @@ module App
28
28
  def start!
29
29
  running!
30
30
  Utility::Logger.info("Starting connector service in #{App::Config.native_mode ? 'native' : 'non-native'} mode...")
31
-
32
- # start sync jobs consumer
33
- start_consumer!
34
-
35
31
  start_polling_jobs!
36
32
  end
37
33
 
@@ -41,8 +37,6 @@ module App
41
37
  scheduler.shutdown
42
38
  pool.shutdown
43
39
  pool.wait_for_termination(TERMINATION_TIMEOUT)
44
-
45
- stop_consumer!
46
40
  end
47
41
 
48
42
  private
@@ -74,16 +68,11 @@ module App
74
68
  scheduler.when_triggered do |connector_settings, task|
75
69
  case task
76
70
  when :sync
77
- # update connector sync_now flag
78
- Core::ElasticConnectorActions.update_connector_sync_now(connector_settings.id, false)
79
-
80
- Core::Jobs::Producer.enqueue_job(job_type: :sync, connector_settings: connector_settings)
71
+ start_sync_task(connector_settings)
81
72
  when :heartbeat
82
73
  start_heartbeat_task(connector_settings)
83
74
  when :configuration
84
75
  start_configuration_task(connector_settings)
85
- when :filter_validation
86
- start_filter_validation_task(connector_settings)
87
76
  else
88
77
  Utility::Logger.error("Unknown task type: #{task}. Skipping...")
89
78
  end
@@ -92,6 +81,22 @@ module App
92
81
  Utility::ExceptionTracking.log_exception(e, 'The connector service failed due to unexpected error.')
93
82
  end
94
83
 
84
+ def start_sync_task(connector_settings)
85
+ start_heartbeat_task(connector_settings)
86
+ pool.post do
87
+ Utility::Logger.info("Initiating a sync job for #{connector_settings.formatted}...")
88
+ Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
89
+ job_runner = Core::SyncJobRunner.new(connector_settings)
90
+ job_runner.execute
91
+ rescue Core::JobAlreadyRunningError
92
+ Utility::Logger.info("Sync job for #{connector_settings.formatted} is already running, skipping.")
93
+ rescue Core::ConnectorVersionChangedError => e
94
+ Utility::Logger.info("Could not start the job because #{connector_settings.formatted} has been updated externally. Message: #{e.message}")
95
+ rescue StandardError => e
96
+ Utility::ExceptionTracking.log_exception(e, "Sync job for #{connector_settings.formatted} failed due to unexpected error.")
97
+ end
98
+ end
99
+
95
100
  def start_heartbeat_task(connector_settings)
96
101
  pool.post do
97
102
  Utility::Logger.info("Sending heartbeat for #{connector_settings.formatted}...")
@@ -115,36 +120,6 @@ module App
115
120
  Utility::ExceptionTracking.log_exception(e, "Configuration task for #{connector_settings.formatted} failed due to unexpected error.")
116
121
  end
117
122
  end
118
-
119
- def start_filter_validation_task(connector_settings)
120
- pool.post do
121
- Utility::Logger.info("Validating filters for #{connector_settings.formatted}...")
122
- validation_job_runner = Core::Filtering::ValidationJobRunner.new(connector_settings)
123
- validation_job_runner.execute
124
- rescue StandardError => e
125
- Utility::ExceptionTracking.log_exception(e, "Filter validation task for #{connector_settings.formatted} failed due to unexpected error.")
126
- end
127
- end
128
-
129
- def start_consumer!
130
- @consumer = Core::Jobs::Consumer.new(
131
- poll_interval: POLL_INTERVAL,
132
- termination_timeout: TERMINATION_TIMEOUT,
133
- min_threads: MIN_THREADS,
134
- max_threads: MAX_THREADS,
135
- max_queue: MAX_QUEUE,
136
- scheduler: scheduler
137
- )
138
-
139
- @consumer.subscribe!(index_name: Utility::Constants::JOB_INDEX)
140
- end
141
-
142
- def stop_consumer!
143
- return if @consumer.nil?
144
- return unless @consumer.running?
145
-
146
- @consumer.shutdown!
147
- end
148
123
  end
149
124
  end
150
125
  end
@@ -23,7 +23,6 @@ module App
23
23
  check_es_connection!
24
24
  check_es_version!
25
25
  check_system_indices!
26
- check_single_connector!
27
26
  end
28
27
 
29
28
  private
@@ -60,16 +59,6 @@ module App
60
59
  )
61
60
  end
62
61
 
63
- #-------------------------------------------------------------------------------------------------
64
- # Ensures the connector is supported when running in non-native mode
65
- def check_single_connector!
66
- if App::Config.native_mode
67
- Utility::Logger.info('Skip single connector check for native mode.')
68
- elsif !Connectors::REGISTRY.registered?(App::Config.service_type)
69
- fail_check!("The service type #{App::Config.service_type} is not supported. Terminating...")
70
- end
71
- end
72
-
73
62
  def check_es_connection_with_retries!(retry_interval:, retry_timeout:)
74
63
  started_at = Time.now
75
64
 
@@ -6,15 +6,11 @@
6
6
 
7
7
  # frozen_string_literal: true
8
8
 
9
- require 'active_support/core_ext/hash/indifferent_access'
10
- require 'app/config'
11
9
  require 'bson'
12
- require 'connectors/base/advanced_snippet_validator'
13
- require 'core/ingestion'
14
- require 'connectors/tolerable_error_helper'
15
- require 'core/filtering/validation_status'
10
+ require 'core/output_sink'
16
11
  require 'utility'
17
- require 'utility/filtering'
12
+ require 'app/config'
13
+ require 'active_support/core_ext/hash/indifferent_access'
18
14
 
19
15
  module Connectors
20
16
  module Base
@@ -36,50 +32,20 @@ module Connectors
36
32
  raise 'Not implemented for this connector'
37
33
  end
38
34
 
39
- def self.kibana_features
40
- [
41
- Utility::Constants::FILTERING_RULES_FEATURE,
42
- Utility::Constants::FILTERING_ADVANCED_FEATURE
43
- ]
44
- end
45
-
46
- def self.advanced_snippet_validator
47
- AdvancedSnippetValidator
48
- end
49
-
50
- def self.validate_filtering(filtering = {})
51
- # nothing to validate
52
- return { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] } unless filtering.present?
53
-
54
- filter = Utility::Filtering.extract_filter(filtering)
55
- advanced_snippet = filter.dig(:advanced_snippet, :value)
56
-
57
- snippet_validator_instance = advanced_snippet_validator.new(advanced_snippet)
58
-
59
- snippet_validator_instance.is_snippet_valid?
60
- end
61
-
62
35
  attr_reader :rules, :advanced_filter_config
63
36
 
64
37
  def initialize(configuration: {}, job_description: {})
65
- error_monitor = Utility::ErrorMonitor.new
66
- @tolerable_error_helper = Connectors::TolerableErrorHelper.new(error_monitor)
67
-
68
38
  @configuration = configuration.dup || {}
69
39
  @job_description = job_description&.dup || {}
70
40
 
71
- filtering = Utility::Filtering.extract_filter(@job_description.dig(:connector, :filtering))
41
+ filter = get_filter(@job_description[:filtering])
72
42
 
73
- @rules = filtering[:rules] || []
74
- @advanced_filter_config = filtering.dig(:advanced_snippet, :value) || {}
43
+ @rules = Utility::Common.return_if_present(filter[:rules], [])
44
+ @advanced_filter_config = Utility::Common.return_if_present(filter[:advanced_config], {})
75
45
  end
76
46
 
77
47
  def yield_documents; end
78
48
 
79
- def yield_with_handling_tolerable_errors(identifier: nil, &block)
80
- @tolerable_error_helper.yield_single_document(identifier: identifier, &block)
81
- end
82
-
83
49
  def do_health_check
84
50
  raise 'Not implemented for this connector'
85
51
  end
@@ -101,11 +67,16 @@ module Connectors
101
67
  end
102
68
 
103
69
  def filtering_present?
104
- @advanced_filter_config.present? && !@advanced_filter_config.empty? || @rules.present?
70
+ @advanced_filter_config.present? || @rules.present?
105
71
  end
106
72
 
107
- def metadata
108
- {}
73
+ private
74
+
75
+ def get_filter(filtering)
76
+ # assume for now, that first object in filtering array or a filter object itself is the only filtering object
77
+ filter = filtering.is_a?(Array) ? filtering[0] : filtering
78
+
79
+ filter.present? ? filter : {}
109
80
  end
110
81
  end
111
82
  end
@@ -7,8 +7,6 @@
7
7
  # frozen_string_literal: true
8
8
 
9
9
  require 'connectors/base/connector'
10
- require 'connectors/example/example_advanced_snippet_validator'
11
- require 'core/filtering/validation_status'
12
10
  require 'utility'
13
11
 
14
12
  module Connectors
@@ -47,10 +45,6 @@ module Connectors
47
45
  # raise 'something went wrong'
48
46
  end
49
47
 
50
- def self.advanced_snippet_validator
51
- ExampleAdvancedSnippetValidator
52
- end
53
-
54
48
  def yield_documents
55
49
  attachments = [
56
50
  load_attachment('first_attachment.txt'),
@@ -11,8 +11,7 @@ require 'connectors/base/connector'
11
11
  require 'connectors/gitlab/extractor'
12
12
  require 'connectors/gitlab/custom_client'
13
13
  require 'connectors/gitlab/adapter'
14
- require 'connectors/gitlab/gitlab_advanced_snippet_validator'
15
- require 'core/ingestion'
14
+ require 'core/output_sink'
16
15
 
17
16
  module Connectors
18
17
  module GitLab
@@ -37,10 +36,6 @@ module Connectors
37
36
  }
38
37
  end
39
38
 
40
- def self.advanced_snippet_validator
41
- GitLabAdvancedSnippetValidator
42
- end
43
-
44
39
  def initialize(configuration: {}, job_description: {})
45
40
  super
46
41
 
@@ -7,9 +7,6 @@
7
7
  # frozen_string_literal: true
8
8
 
9
9
  require 'connectors/base/connector'
10
- require 'core/filtering/validation_status'
11
- require 'connectors/mongodb/mongo_rules_parser'
12
- require 'connectors/mongodb/mongo_advanced_snippet_against_schema_validator'
13
10
  require 'mongo'
14
11
  require 'utility'
15
12
 
@@ -31,31 +28,27 @@ module Connectors
31
28
 
32
29
  def self.configurable_fields
33
30
  {
34
- :host => {
35
- :label => 'Server Hostname'
36
- },
37
- :user => {
38
- :label => 'Username'
39
- },
40
- :password => {
41
- :label => 'Password'
42
- },
43
- :database => {
44
- :label => 'Database'
45
- },
46
- :collection => {
47
- :label => 'Collection'
48
- },
49
- :direct_connection => {
50
- :label => 'Direct connection? (true/false)'
51
- }
31
+ :host => {
32
+ :label => 'Server Hostname'
33
+ },
34
+ :user => {
35
+ :label => 'Username'
36
+ },
37
+ :password => {
38
+ :label => 'Password'
39
+ },
40
+ :database => {
41
+ :label => 'Database'
42
+ },
43
+ :collection => {
44
+ :label => 'Collection'
45
+ },
46
+ :direct_connection => {
47
+ :label => 'Direct connection? (true/false)'
48
+ }
52
49
  }
53
50
  end
54
51
 
55
- def self.advanced_snippet_validator
56
- MongoAdvancedSnippetAgainstSchemaValidator
57
- end
58
-
59
52
  def initialize(configuration: {}, job_description: {})
60
53
  super
61
54
 
@@ -68,6 +61,8 @@ module Connectors
68
61
  end
69
62
 
70
63
  def yield_documents
64
+ check_filtering
65
+
71
66
  with_client do |client|
72
67
  # We do paging using skip().limit() here to make Ruby recycle the memory for each page pulled from the server after it's not needed any more.
73
68
  # This gives us more control on the usage of the memory (we can adjust PAGE_SIZE constant for that to decrease max memory consumption).
@@ -94,15 +89,15 @@ module Connectors
94
89
  loop do
95
90
  found_in_page = 0
96
91
 
97
- Utility::Logger.info("Requesting #{PAGE_SIZE} documents from MongoDB (Starting at #{skip})")
98
92
  view = cursor.skip(skip).limit(PAGE_SIZE)
99
93
  view.each do |document|
100
- yield_with_handling_tolerable_errors do
101
- yield serialize(document)
102
- found_in_page += 1
103
- found_overall += 1
104
- overall_limit_reached = found_overall >= overall_limit && overall_limit != Float::INFINITY
105
- end
94
+ yield serialize(document)
95
+
96
+ found_in_page += 1
97
+ found_overall += 1
98
+
99
+ overall_limit_reached = found_overall >= overall_limit && overall_limit != Float::INFINITY
100
+
106
101
  break if overall_limit_reached
107
102
  end
108
103
 
@@ -122,11 +117,22 @@ module Connectors
122
117
 
123
118
  return create_aggregate_cursor(collection) if @advanced_filter_config[:aggregate].present?
124
119
 
125
- return create_simple_rules_cursor(collection) if @rules.present?
126
-
127
120
  collection.find
128
121
  end
129
122
 
123
+ def check_filtering
124
+ return unless filtering_present?
125
+
126
+ check_find_and_aggregate
127
+ end
128
+
129
+ def check_find_and_aggregate
130
+ if @advanced_filter_config.keys.size != 1
131
+ invalid_keys_msg = "Only one of #{ALLOWED_TOP_LEVEL_FILTER_KEYS} is allowed in the filtering object. Keys present: '#{@advanced_filter_config.keys}'."
132
+ raise Utility::InvalidFilterConfigError.new(invalid_keys_msg)
133
+ end
134
+ end
135
+
130
136
  def create_aggregate_cursor(collection)
131
137
  aggregate = @advanced_filter_config[:aggregate]
132
138
 
@@ -153,16 +159,6 @@ module Connectors
153
159
  [collection.find(filter, options), options]
154
160
  end
155
161
 
156
- def create_simple_rules_cursor(collection)
157
- filter = {}
158
- if @rules.present?
159
- parser = MongoRulesParser.new(@rules)
160
- filter = parser.parse
161
- end
162
- Utility::Logger.info("Filtering with simple rules filter: #{filter}")
163
- filter.present? ? collection.find(filter) : collection.find
164
- end
165
-
166
162
  def extract_options(mongodb_function)
167
163
  mongodb_function[:options].present? ? mongodb_function[:options] : {}
168
164
  end
@@ -177,9 +173,9 @@ module Connectors
177
173
  raise "Invalid value for 'Direct connection' : #{@direct_connection}." unless %w[true false].include?(@direct_connection.to_s.strip.downcase)
178
174
 
179
175
  args = {
180
- database: @database,
181
- direct_connection: to_boolean(@direct_connection)
182
- }
176
+ database: @database,
177
+ direct_connection: to_boolean(@direct_connection)
178
+ }
183
179
 
184
180
  if @user.present? || @password.present?
185
181
  args[:user] = @user
@@ -26,16 +26,11 @@ module Connectors
26
26
  ERROR
27
27
  ]
28
28
 
29
- PENDING_STATUSES = [
29
+ PENDING_STATUES = [
30
30
  PENDING,
31
31
  SUSPENDED
32
32
  ]
33
33
 
34
- ACTIVE_STATUSES = [
35
- IN_PROGRESS,
36
- CANCELING
37
- ]
38
-
39
34
  TERMINAL_STATUSES = [
40
35
  CANCELED,
41
36
  COMPLETED,
@@ -24,10 +24,8 @@ module Core
24
24
  return
25
25
  end
26
26
  configuration = connector_class.configurable_fields_indifferent_access
27
- features = connector_class.kibana_features.each_with_object({}) { |feature, hsh| hsh[feature] = true }
28
27
  doc = {
29
- :configuration => configuration,
30
- :features => features
28
+ :configuration => configuration
31
29
  }
32
30
 
33
31
  doc[:service_type] = service_type if service_type && connector_settings.needs_service_type?
@@ -23,24 +23,24 @@ module Core
23
23
 
24
24
  DEFAULT_PAGE_SIZE = 100
25
25
 
26
+ # Error Classes
27
+ class ConnectorNotFoundError < StandardError; end
28
+
26
29
  def self.fetch_by_id(connector_id)
27
30
  es_response = ElasticConnectorActions.get_connector(connector_id)
28
- return nil unless es_response[:found]
29
-
30
31
  connectors_meta = ElasticConnectorActions.connectors_meta
32
+
33
+ raise ConnectorNotFoundError.new("Connector with id=#{connector_id} was not found.") unless es_response[:found]
31
34
  new(es_response, connectors_meta)
32
35
  end
33
36
 
37
+ def initialize(es_response, connectors_meta)
38
+ @elasticsearch_response = es_response.with_indifferent_access
39
+ @connectors_meta = connectors_meta.with_indifferent_access
40
+ end
41
+
34
42
  def self.fetch_native_connectors(page_size = DEFAULT_PAGE_SIZE)
35
- require 'connectors/registry' unless defined?(Connectors::REGISTRY)
36
- query = {
37
- bool: {
38
- filter: [
39
- { term: { is_native: true } },
40
- { terms: { service_type: Connectors::REGISTRY.registered_connectors } }
41
- ]
42
- }
43
- }
43
+ query = { term: { is_native: true } }
44
44
  fetch_connectors_by_query(query, page_size)
45
45
  end
46
46
 
@@ -83,26 +83,23 @@ module Core
83
83
  end
84
84
 
85
85
  def filtering
86
- # assume for now, that first object in filtering array or a filter object itself is the only filtering object
87
- filtering = @elasticsearch_response.dig(:_source, :filtering)
88
-
89
- Utility::Filtering.extract_filter(filtering)
86
+ Utility::Common.return_if_present(@elasticsearch_response[:filtering], DEFAULT_FILTERING)
90
87
  end
91
88
 
92
89
  def request_pipeline
93
- Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
90
+ Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
94
91
  end
95
92
 
96
93
  def extract_binary_content?
97
- Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
94
+ Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
98
95
  end
99
96
 
100
97
  def reduce_whitespace?
101
- Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
98
+ Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
102
99
  end
103
100
 
104
101
  def run_ml_inference?
105
- Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
102
+ Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
106
103
  end
107
104
 
108
105
  def formatted
@@ -119,39 +116,6 @@ module Core
119
116
  index_name&.start_with?(Utility::Constants::CONTENT_INDEX_PREFIX)
120
117
  end
121
118
 
122
- def ready_for_sync?
123
- Connectors::REGISTRY.registered?(service_type) &&
124
- valid_index_name? &&
125
- connector_status_allows_sync?
126
- end
127
-
128
- def running?
129
- @elasticsearch_response[:_source][:last_sync_status] == Connectors::SyncStatus::IN_PROGRESS
130
- end
131
-
132
- def update_last_sync!(job)
133
- doc = {
134
- :last_sync_status => job.status,
135
- :last_synced => Time.now,
136
- :last_sync_error => job.error,
137
- :error => job.error
138
- }
139
-
140
- if job.terminated?
141
- doc[:last_indexed_document_count] = job[:indexed_document_count]
142
- doc[:last_deleted_document_count] = job[:deleted_document_count]
143
- end
144
-
145
- Core::ElasticConnectorActions.update_connector_fields(job.connector_id, doc)
146
- end
147
-
148
- private
149
-
150
- def initialize(es_response, connectors_meta)
151
- @elasticsearch_response = es_response.with_indifferent_access
152
- @connectors_meta = connectors_meta.with_indifferent_access
153
- end
154
-
155
119
  def self.fetch_connectors_by_query(query, page_size)
156
120
  connectors_meta = ElasticConnectorActions.connectors_meta
157
121