connectors_utility 8.6.0.3 → 8.6.0.4.pre.20221104T201057Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb8d3022cb5894d77ba44a62be03bea6d9a944a49c97a9dcd87aa92f37ae418c
-  data.tar.gz: 822efb10e7fed04267461bb1515d08ad6e9831c4c79add93391dd457e3961eeb
+  metadata.gz: 75130f5f1747db0119b4619066fa71a9502d82c86838966c9453541704dc5e04
+  data.tar.gz: a1dfebbed9be1639600cce8f35f49800d9a5538e97a4ed57d8674d81796902d1
 SHA512:
-  metadata.gz: e90c74ffd7efd454e855437ec51ef4c00620665a7dd46a131d37631a68f69c170bb149177619f10c825a6ea447483294e02be9c613d85c689822dfa7a78e42a2
-  data.tar.gz: 7169e3d728d9eadd5441f38029137e6fd5706b6f5d06a1241419d91d89b130f5289ac8eccdd3ecd6b8b14706b89a8783f4d1029186788500a99c4822914d3301
+  metadata.gz: 6faaf8c6151c35c0923304cd11692a03bca2cf4f8f0544d38a76e30eb90ec3090e5f8b5a8249d1bd4f4724c0a4712b1ffc58a1d8be1d46fa18e0a1d1f5e2e045
+  data.tar.gz: 57de1647be647f92b9934445034203ed762a453f62e5ce1b4554dff2c1632370eb86bd3b7ff44b2a6380d4ab75f691e8c0c70a39e5caecdfee27f876c475a73f

data/lib/connectors/connector_status.rb

@@ -8,11 +8,11 @@
 
 module Connectors
   class ConnectorStatus
-    CREATED = 'created'
+    CREATED             = 'created'
     NEEDS_CONFIGURATION = 'needs_configuration'
-    CONFIGURED = 'configured'
-    CONNECTED = 'connected'
-    ERROR = 'error'
+    CONFIGURED          = 'configured'
+    CONNECTED           = 'connected'
+    ERROR               = 'error'
 
     STATUSES = [
       CREATED,

data/lib/connectors/sync_status.rb

@@ -8,14 +8,33 @@
 
 module Connectors
   class SyncStatus
-    COMPLETED = 'completed'
+    PENDING     = 'pending'
     IN_PROGRESS = 'in_progress'
-    FAILED = 'failed'
+    CANCELING   = 'canceling'
+    CANCELED    = 'canceled'
+    SUSPENDED   = 'suspended'
+    COMPLETED   = 'completed'
+    ERROR       = 'error'
 
     STATUSES = [
-      COMPLETED,
+      PENDING,
       IN_PROGRESS,
-      FAILED
+      CANCELING,
+      CANCELED,
+      SUSPENDED,
+      COMPLETED,
+      ERROR
+    ]
+
+    PENDING_STATUES = [
+      PENDING,
+      SUSPENDED
+    ]
+
+    TERMINAL_STATUSES = [
+      CANCELED,
+      COMPLETED,
+      ERROR
     ]
   end
 end

data/lib/core/connector_settings.rb

@@ -19,6 +19,8 @@ module Core
     DEFAULT_REDUCE_WHITESPACE = true
     DEFAULT_RUN_ML_INFERENCE = true
 
+    DEFAULT_FILTERING = {}
+
     DEFAULT_PAGE_SIZE = 100
 
     # Error Classes
@@ -80,20 +82,24 @@ module Core
       self[:scheduling]
     end
 
+    def filtering
+      Utility::Common.return_if_present(@elasticsearch_response[:filtering], DEFAULT_FILTERING)
+    end
+
     def request_pipeline
-      return_if_present(@elasticsearch_response.dig(:pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
     end
 
     def extract_binary_content?
-      return_if_present(@elasticsearch_response.dig(:pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
     end
 
     def reduce_whitespace?
-      return_if_present(@elasticsearch_response.dig(:pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
     end
 
     def run_ml_inference?
-      return_if_present(@elasticsearch_response.dig(:pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
     end
 
     def formatted
@@ -110,8 +116,6 @@ module Core
       index_name&.start_with?(Utility::Constants::CONTENT_INDEX_PREFIX)
     end
 
-    private
-
     def self.fetch_connectors_by_query(query, page_size)
       connectors_meta = ElasticConnectorActions.connectors_meta
 
@@ -120,8 +124,8 @@ module Core
       loop do
         response = ElasticConnectorActions.search_connectors(query, page_size, offset)
 
-        hits = response['hits']['hits']
-        total = response['hits']['total']['value']
+        hits = response.dig('hits', 'hits') || []
+        total = response.dig('hits', 'total', 'value') || 0
         results += hits.map do |hit|
           Core::ConnectorSettings.new(hit, connectors_meta)
         end
@@ -132,11 +136,5 @@ module Core
       results
     end
 
-    def return_if_present(*args)
-      args.each do |arg|
-        return arg unless arg.nil?
-      end
-      nil
-    end
   end
 end

data/lib/core/elastic_connector_actions.rb

@@ -10,8 +10,21 @@ require 'active_support/core_ext/hash'
 require 'connectors/connector_status'
 require 'connectors/sync_status'
 require 'utility'
+require 'elastic-transport'
 
 module Core
+  class JobAlreadyRunningError < StandardError
+    def initialize(connector_id)
+      super("Sync job for connector '#{connector_id}' is already running.")
+    end
+  end
+
+  class ConnectorVersionChangedError < StandardError
+    def initialize(connector_id, seq_no, primary_term)
+      super("Version conflict: seq_no [#{seq_no}] and primary_term [#{primary_term}] do not match for connector '#{connector_id}'.")
+    end
+  end
+
   class ElasticConnectorActions
     class << self
 
@@ -72,20 +85,53 @@ module Core
       end
 
       def claim_job(connector_id)
-        update_connector_fields(connector_id,
-                                :sync_now => false,
-                                :last_sync_status => Connectors::SyncStatus::IN_PROGRESS,
-                                :last_synced => Time.now)
+        seq_no = nil
+        primary_term = nil
+        sync_in_progress = false
+        connector_record = client.get(
+          :index => Utility::Constants::CONNECTORS_INDEX,
+          :id => connector_id,
+          :ignore => 404,
+          :refresh => true
+        ).tap do |response|
+          seq_no = response['_seq_no']
+          primary_term = response['_primary_term']
+          sync_in_progress = response.dig('_source', 'last_sync_status') == Connectors::SyncStatus::IN_PROGRESS
+        end
+        if sync_in_progress
+          raise JobAlreadyRunningError.new(connector_id)
+        end
+        update_connector_fields(
+          connector_id,
+          { :sync_now => false,
+            :last_sync_status => Connectors::SyncStatus::IN_PROGRESS,
+            :last_synced => Time.now },
+          seq_no,
+          primary_term
+        )
 
         body = {
           :connector_id => connector_id,
           :status => Connectors::SyncStatus::IN_PROGRESS,
           :worker_hostname => Socket.gethostname,
-          :created_at => Time.now
+          :created_at => Time.now,
+          :filtering => convert_connector_filtering_to_job_filtering(connector_record.dig('_source', 'filtering'))
         }
-        job = client.index(:index => Utility::Constants::JOB_INDEX, :body => body)
 
-        job['_id']
+        client.index(:index => Utility::Constants::JOB_INDEX, :body => body)
+      end
+
+      def convert_connector_filtering_to_job_filtering(connector_filtering)
+        return [] unless connector_filtering
+        connector_filtering = [connector_filtering] unless connector_filtering.is_a?(Array)
+        connector_filtering.each_with_object([]) do |filtering_domain, job_filtering|
+          job_filtering << {
+            'domain' => filtering_domain['domain'],
+            'rules' => filtering_domain.dig('active', 'rules'),
+            'advanced_snippet' => filtering_domain.dig('active', 'advanced_snippet'),
+            'warnings' => [] # TODO: in https://github.com/elastic/enterprise-search-team/issues/3174
+          }
+        end
       end
 
       def update_connector_status(connector_id, status, error_message = nil)
@@ -100,7 +146,7 @@ module Core
       end
 
       def complete_sync(connector_id, job_id, status)
-        sync_status = status[:error] ? Connectors::SyncStatus::FAILED : Connectors::SyncStatus::COMPLETED
+        sync_status = status[:error] ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
 
         update_connector_fields(connector_id,
                                 :last_sync_status => sync_status,
@@ -136,7 +182,7 @@ module Core
           }
           loop do
             response = client.search(:body => body)
-            hits = response['hits']['hits']
+            hits = response.dig('hits', 'hits') || []
 
             ids = hits.map { |h| h['_id'] }
             result += ids
@@ -242,15 +288,29 @@ module Core
         ensure_index_exists("#{Utility::Constants::JOB_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::JOB_INDEX, :mappings => mappings))
       end
 
-      def update_connector_fields(connector_id, doc = {})
+      def update_connector_fields(connector_id, doc = {}, seq_no = nil, primary_term = nil)
         return if doc.empty?
-        client.update(
+        update_args = {
           :index => Utility::Constants::CONNECTORS_INDEX,
           :id => connector_id,
           :body => { :doc => doc },
           :refresh => true,
           :retry_on_conflict => 3
-        )
+        }
+        # seq_no and primary_term are used for optimistic concurrency control
+        # see https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html
+        if seq_no && primary_term
+          update_args[:if_seq_no] = seq_no
+          update_args[:if_primary_term] = primary_term
+          update_args.delete(:retry_on_conflict)
+        end
+        begin
+          client.update(update_args)
+        rescue Elastic::Transport::Transport::Errors::Conflict
+          # VersionConflictException
+          # see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#optimistic-concurrency-control-index
+          raise ConnectorVersionChangedError.new(connector_id, seq_no, primary_term)
+        end
       end
 
       private

data/lib/core/scheduler.rb

@@ -9,6 +9,7 @@
 require 'time'
 require 'fugit'
 require 'core/connector_settings'
+require 'core/elastic_connector_actions'
 require 'utility/cron'
 require 'utility/logger'
 require 'utility/exception_tracking'
@@ -41,6 +42,8 @@ module Core
         if @is_shutting_down
           break
         end
+      rescue *Utility::AUTHORIZATION_ERRORS => e
+        Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
       rescue StandardError => e
         Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
       ensure

data/lib/utility/errors.rb

@@ -5,6 +5,7 @@
 #
 
 require 'active_support/core_ext/string'
+require 'elasticsearch'
 
 module Utility
   class DocumentError
@@ -31,6 +32,8 @@ module Utility
   end
 
   class ClientError < StandardError; end
+
+  class InvalidFilterConfigError < StandardError; end
   class EvictionWithNoProgressError < StandardError; end
   class EvictionError < StandardError
     attr_accessor :cursors
@@ -89,6 +92,7 @@ module Utility
   class InvalidTokenError < StandardError; end
   class TokenRefreshFailedError < StandardError; end
   class ConnectorNotAvailableError < StandardError; end
+  class AuthorizationError < StandardError; end
 
   # For when we want to explicitly set a #cause but can't
   class ExplicitlyCausedError < StandardError
@@ -124,6 +128,7 @@ module Utility
     end
   end
 
+  AUTHORIZATION_ERRORS = [Elastic::Transport::Transport::Errors::Unauthorized]
   INTERNAL_SERVER_ERROR = Utility::Error.new(500, 'INTERNAL_SERVER_ERROR', 'Internal server error')
   INVALID_API_KEY = Utility::Error.new(401, 'INVALID_API_KEY', 'Invalid API key')
   UNSUPPORTED_AUTH_SCHEME = Utility::Error.new(401, 'UNSUPPORTED_AUTH_SCHEME', 'Unsupported authorization scheme')

data/lib/utility/es_client.rb

@@ -20,8 +20,8 @@ module Utility
       attr_reader :cause
     end
 
-    def initialize(es_config)
-      super(connection_configs(es_config))
+    def initialize(es_config, &block)
+      super(connection_configs(es_config), &block)
     end
 
     def connection_configs(es_config)
@@ -39,6 +39,10 @@ module Utility
       configs[:log] = es_config[:log] || false
       configs[:trace] = es_config[:trace] || false
 
+      # transport options
+      configs[:transport_options] = es_config[:transport_options] if es_config[:transport_options]
+      configs[:ca_fingerprint] = es_config[:ca_fingerprint] if es_config[:ca_fingerprint]
+
       # if log or trace is activated, we use the application logger
       configs[:logger] = if configs[:log] || configs[:trace]
                            Utility::Logger.logger

data/lib/utility.rb

@@ -6,6 +6,7 @@
 
 require 'utility/constants'
 require 'utility/cron'
+require 'utility/common'
 require 'utility/errors'
 require 'utility/es_client'
 require 'utility/environment'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_utility
 version: !ruby/object:Gem::Version
-  version: 8.6.0.3
+  version: 8.6.0.4.pre.20221104T201057Z
 platform: ruby
 authors:
 - Elastic
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-10-03 00:00:00.000000000 Z
+date: 2022-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -125,9 +125,9 @@ homepage: https://github.com/elastic/connectors-ruby
 licenses:
 - Elastic-2.0
 metadata:
-  revision: aa2faf8cc993a26980441adffe97d62fdaf5aa5c
-  repository: https://github.com/elastic/connectors-ruby.git
-post_install_message:
+  revision: 2051b3907639a1fe2ae68efdc33c06cf12d38383
+  repository: git@github.com:elastic/ent-search-connectors.git
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -138,12 +138,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3.1
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Gem containing shared Connector Services libraries
 test_files: []