connectors_utility 8.6.0.4.pre.20221115T002329Z → 8.7.0.0.pre.20221117T004939Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 996cc59bf15e82bf245bc30716d0ac93861514fe9f8e73f92948f80e60adccc4
-  data.tar.gz: f904929c8d82a73e1763cf30536c7e703b885d633f05e9cdecbba310eb44d99d
+  metadata.gz: 5d2972f8e6974a79b6088ce6c03453c327132ce19ffb09dbf30f349eae4c2108
+  data.tar.gz: 4fd458de07be07923e0675dc0f341b8211ba3daeec8ac27bfb4f9eb9aff2334a
 SHA512:
-  metadata.gz: d22401f2411e468c734bb015e6027ede2858a64fcdeb6ea446ad9cc4871c68791e736dea230e5a2fcee2c77b84514c41314cd866f7766bb2906e4396b1c557e8
-  data.tar.gz: e3ed8f7cb8e0d10bfbc220520ff7d98dcc7d221ee65f4790976eb6e1f538f78ac781eaa453258896749ed56e41eb3cf3c065a206cbb209740d43dbe6b37f7b52
+  metadata.gz: 9db02a3003d5645cbb5d57d4ca1bdb1acb65234f5a931afd9ccb06e2fbbe25be2394c65a72a9ae36038c8c127c35a0d937c83558ede0e4960fc688b073db052a
+  data.tar.gz: d02681e0d4009420b949ec649c9eac52bd533eee493c181ea1ffb15d939b561e81f234bac4885eefd0ea82c8564d7d330c0635c535b0616cfb9280e2e38512df

data/lib/connectors/sync_status.rb

@@ -26,11 +26,16 @@ module Connectors
       ERROR
     ]
 
-    PENDING_STATUES = [
+    PENDING_STATUSES = [
       PENDING,
       SUSPENDED
     ]
 
+    ACTIVE_STATUSES = [
+      IN_PROGRESS,
+      CANCELING
+    ]
+
     TERMINAL_STATUSES = [
       CANCELED,
       COMPLETED,

data/lib/core/connector_settings.rb

@@ -23,14 +23,11 @@ module Core
 
     DEFAULT_PAGE_SIZE = 100
 
-    # Error Classes
-    class ConnectorNotFoundError < StandardError; end
-
     def self.fetch_by_id(connector_id)
       es_response = ElasticConnectorActions.get_connector(connector_id)
-      connectors_meta = ElasticConnectorActions.connectors_meta
+      return nil unless es_response[:found]
 
-      raise ConnectorNotFoundError.new("Connector with id=#{connector_id} was not found.") unless es_response[:found]
+      connectors_meta = ElasticConnectorActions.connectors_meta
       new(es_response, connectors_meta)
     end
 
@@ -122,6 +119,32 @@ module Core
       index_name&.start_with?(Utility::Constants::CONTENT_INDEX_PREFIX)
     end
 
+    def ready_for_sync?
+      Connectors::REGISTRY.registered?(service_type) &&
+        valid_index_name? &&
+        connector_status_allows_sync?
+    end
+
+    def running?
+      @elasticsearch_response[:_source][:last_sync_status] == Connectors::SyncStatus::IN_PROGRESS
+    end
+
+    def update_last_sync!(job)
+      doc = {
+        :last_sync_status => job.status,
+        :last_synced => Time.now,
+        :last_sync_error => job.error,
+        :error => job.error
+      }
+
+      if job.terminated?
+        doc[:last_indexed_document_count] = job[:indexed_document_count]
+        doc[:last_deleted_document_count] = job[:deleted_document_count]
+      end
+
+      Core::ElasticConnectorActions.update_connector_fields(job.connector_id, doc)
+    end
+
     private
 
     def initialize(es_response, connectors_meta)

data/lib/core/elastic_connector_actions.rb

@@ -132,11 +132,35 @@ module Core
         update_connector_fields(connector_id, { :filtering => filtering })
       end
 
-      def claim_job(connector_id)
+      def update_connector_sync_now(connector_id, sync_now)
+        doc = connector_with_concurrency_control(connector_id)
+
+        body = { sync_now: sync_now, last_synced: Time.now }
+
+        update_connector_fields(
+          connector_id,
+          body,
+          doc[:seq_no],
+          doc[:primary_term]
+        )
+      end
+
+      def update_connector_last_sync_status(connector_id, last_sync_status)
+        doc = connector_with_concurrency_control(connector_id)
+
+        update_connector_fields(
+          connector_id,
+          { last_sync_status: last_sync_status },
+          doc[:seq_no],
+          doc[:primary_term]
+        )
+      end
+
+      def connector_with_concurrency_control(connector_id)
         seq_no = nil
         primary_term = nil
-        sync_in_progress = false
-        connector_record = client.get(
+
+        doc = client.get(
           :index => Utility::Constants::CONNECTORS_INDEX,
           :id => connector_id,
           :ignore => 404,
@@ -144,42 +168,31 @@ module Core
         ).tap do |response|
           seq_no = response['_seq_no']
           primary_term = response['_primary_term']
-          sync_in_progress = response.dig('_source', 'last_sync_status') == Connectors::SyncStatus::IN_PROGRESS
-        end
-        if sync_in_progress
-          raise JobAlreadyRunningError.new(connector_id)
         end
-        update_connector_fields(
-          connector_id,
-          { :sync_now => false,
-            :last_sync_status => Connectors::SyncStatus::IN_PROGRESS,
-            :last_synced => Time.now },
-          seq_no,
-          primary_term
-        )
 
+        { doc: doc, seq_no: seq_no, primary_term: primary_term }
+      end
+
+      def create_job(connector_settings:)
         body = {
-          :status => Connectors::SyncStatus::IN_PROGRESS,
-          :worker_hostname => Socket.gethostname,
-          :created_at => Time.now,
-          :started_at => Time.now,
-          :last_seen => Time.now,
-          :connector => {
-            :id => connector_id,
-            :filtering => convert_connector_filtering_to_job_filtering(connector_record.dig('_source', 'filtering'))
+          status: Connectors::SyncStatus::PENDING,
+          created_at: Time.now,
+          last_seen: Time.now,
+          connector: {
+            id: connector_settings.id,
+            filtering: convert_connector_filtering_to_job_filtering(connector_settings.filtering),
+            index_name: connector_settings.index_name,
+            language: connector_settings[:language],
+            pipeline: connector_settings[:pipeline],
+            service_type: connector_settings.service_type
           }
         }
 
-        index_response = client.index(:index => Utility::Constants::JOB_INDEX, :body => body, :refresh => true)
-        if index_response['result'] == 'created'
-          # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
-          return client.get(
-            :index => Utility::Constants::JOB_INDEX,
-            :id => index_response['_id'],
-            :ignore => 404
-          ).with_indifferent_access
-        end
-        raise JobNotCreatedError.new(connector_id, index_response)
+        index_response = client.index(index: Utility::Constants::JOB_INDEX, body: body, refresh: true)
+
+        return index_response if index_response['result'] == 'created'
+
+        raise JobNotCreatedError.new(connector_settings.id, index_response)
       end
 
       def convert_connector_filtering_to_job_filtering(connector_filtering)
@@ -507,31 +520,15 @@ module Core
       end
 
       def update_connector_fields(connector_id, doc = {}, seq_no = nil, primary_term = nil)
-        return if doc.empty?
-        update_args = {
-          :index => Utility::Constants::CONNECTORS_INDEX,
-          :id => connector_id,
-          :body => { :doc => doc },
-          :refresh => true,
-          :retry_on_conflict => 3
-        }
-        # seq_no and primary_term are used for optimistic concurrency control
-        # see https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html
-        if seq_no && primary_term
-          update_args[:if_seq_no] = seq_no
-          update_args[:if_primary_term] = primary_term
-          update_args.delete(:retry_on_conflict)
-        end
-        begin
-          client.update(update_args)
-        rescue Elastic::Transport::Transport::Errors::Conflict
-          # VersionConflictException
-          # see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#optimistic-concurrency-control-index
-          raise ConnectorVersionChangedError.new(connector_id, seq_no, primary_term)
-        end
+        update_doc_fields(Utility::Constants::CONNECTORS_INDEX, connector_id, doc, seq_no, primary_term)
+      end
+
+      def update_job_fields(job_id, doc = {}, seq_no = nil, primary_term = nil)
+        update_doc_fields(Utility::Constants::JOB_INDEX, job_id, doc, seq_no, primary_term)
       end
 
       def document_count(index_name)
+        client.indices.refresh(:index => index_name)
         client.count(:index => index_name)['count']
       end
 
@@ -563,6 +560,31 @@ module Core
           filter.deep_merge!(new_validation_state)
         end
       end
+
+      def update_doc_fields(index, id, doc = {}, seq_no = nil, primary_term = nil)
+        return if doc.empty?
+        update_args = {
+          :index => index,
+          :id => id,
+          :body => { :doc => doc },
+          :refresh => true,
+          :retry_on_conflict => 3
+        }
+
+        if seq_no && primary_term
+          update_args[:if_seq_no] = seq_no
+          update_args[:if_primary_term] = primary_term
+          update_args.delete(:retry_on_conflict)
+        end
+
+        begin
+          client.update(update_args)
+        rescue Elastic::Transport::Transport::Errors::Conflict
+          # VersionConflictException
+          # see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#optimistic-concurrency-control-index
+          raise ConnectorVersionChangedError.new(id, seq_no, primary_term)
+        end
+      end
     end
   end
 end

data/lib/utility/bulk_queue.rb

@@ -6,12 +6,14 @@
 
 require 'json'
 
+require 'utility/constants'
+
 module Utility
   class BulkQueue
     class QueueOverflowError < StandardError; end
 
     # 500 items or 5MB
-    def initialize(operation_count_threshold = 500, size_threshold = 5 * 1024 * 1024)
+    def initialize(operation_count_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_LENGTH, size_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES)
       @operation_count_threshold = operation_count_threshold.freeze
       @size_threshold = size_threshold.freeze
 

data/lib/utility/constants.rb

@@ -18,5 +18,10 @@ module Utility
     CRAWLER_SERVICE_TYPE = 'elastic-crawler'
     FILTERING_RULES_FEATURE = 'filtering_rules'
     FILTERING_ADVANCED_FEATURE = 'filtering_advanced_config'
+
+    # Maximum number of operations in BULK Elasticsearch operation that will ingest the data
+    DEFAULT_MAX_INGESTION_QUEUE_SIZE = 500
+    # Maximum size of either whole BULK Elasticsearch operation or one document in it
+    DEFAULT_MAX_INGESTION_QUEUE_BYTES = 5 * 1024 * 1024
   end
 end

data/lib/utility/error_monitor.rb

@@ -0,0 +1,108 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'time'
+require 'utility/errors'
+require 'utility/exception_tracking'
+
+module Utility
+  class ErrorMonitor
+    class MonitoringError < StandardError
+      attr_accessor :tripped_by
+
+      def initialize(message = nil, tripped_by: nil)
+        super("#{message}#{tripped_by.present? ? " Tripped by - #{tripped_by.class}: #{tripped_by.message}" : ''}")
+        @tripped_by = tripped_by
+      end
+    end
+
+    class MaxSuccessiveErrorsExceededError < MonitoringError; end
+    class MaxErrorsExceededError < MonitoringError; end
+    class MaxErrorsInWindowExceededError < MonitoringError; end
+
+    attr_reader :total_error_count, :success_count, :consecutive_error_count, :error_queue
+
+    def initialize(
+      max_errors: 1000,
+      max_consecutive_errors: 10,
+      max_error_ratio: 0.15,
+      window_size: 100,
+      error_queue_size: 20
+    )
+      @max_errors = max_errors
+      @max_consecutive_errors = max_consecutive_errors
+      @max_error_ratio = max_error_ratio
+      @window_size = window_size
+      @total_error_count = 0
+      @success_count = 0
+      @consecutive_error_count = 0
+      @window_errors = Array.new(window_size) { false }
+      @window_index = 0
+      @last_error = nil
+      @error_queue_size = error_queue_size
+      @error_queue = []
+    end
+
+    def note_success
+      @consecutive_error_count = 0
+      @success_count += 1
+      increment_window_index
+    end
+
+    def note_error(error, id: Time.now.to_i)
+      stack_trace = Utility::ExceptionTracking.generate_stack_trace(error)
+      error_message = Utility::ExceptionTracking.generate_error_message(error, nil, nil)
+      Utility::Logger.debug("Message id: #{id} - #{error_message}\n#{stack_trace}")
+      @total_error_count += 1
+      @consecutive_error_count += 1
+      @window_errors[@window_index] = true
+      @error_queue << DocumentError.new(error.class.name, error_message, stack_trace, id)
+      @error_queue = @error_queue.drop(1) if @error_queue.size > @error_queue_size
+      increment_window_index
+      @last_error = error
+
+      raise_if_necessary
+    end
+
+    def finalize
+      total_documents = @total_error_count + @success_count
+      if total_documents > 0 && @total_error_count.to_f / total_documents > @max_error_ratio
+        raise_with_last_cause(MaxErrorsInWindowExceededError.new("There were #{@total_error_count} errors out of #{total_documents} total documents", :tripped_by => @last_error))
+      end
+    end
+
+    private
+
+    def raise_if_necessary
+      error =
+        if @consecutive_error_count > @max_consecutive_errors
+          MaxSuccessiveErrorsExceededError.new("Exceeded maximum consecutive errors - saw #{@consecutive_error_count} errors in a row.", :tripped_by => @last_error)
+        elsif @total_error_count > @max_errors
+          MaxErrorsExceededError.new("Exceeded maximum number of errors - saw #{@total_error_count} errors in total.", :tripped_by => @last_error)
+        elsif @window_size > 0 && num_errors_in_window / @window_size > @max_error_ratio
+          MaxErrorsInWindowExceededError.new("Exceeded maximum error ratio of #{@max_error_ratio}. Of the last #{@window_size} documents, #{num_errors_in_window} had errors", :tripped_by => @last_error)
+        end
+
+      raise_with_last_cause(error) if error
+    end
+
+    def num_errors_in_window
+      @window_errors.count(&:itself).to_f
+    end
+
+    def increment_window_index
+      @window_index = (@window_index + 1) % @window_size
+    end
+
+    def raise_with_last_cause(error)
+      raise @last_error
+    rescue StandardError
+      raise error
+    end
+  end
+end

data/lib/utility/errors.rb

@@ -60,18 +60,6 @@ module Utility
   class JobDocumentLimitError < StandardError; end
   class JobClaimingError < StandardError; end
 
-  class MonitoringError < StandardError
-    attr_accessor :tripped_by
-
-    def initialize(message = nil, tripped_by: nil)
-      super("#{message}#{tripped_by.present? ? " Tripped by - #{tripped_by.class}: #{tripped_by.message}" : ''}")
-      @tripped_by = tripped_by
-    end
-  end
-  class MaxSuccessiveErrorsExceededError < MonitoringError; end
-  class MaxErrorsExceededError < MonitoringError; end
-  class MaxErrorsInWindowExceededError < MonitoringError; end
-
   class JobSyncNotPossibleYetError < StandardError
     attr_accessor :sync_will_be_possible_at
 

data/lib/utility.rb

@@ -4,6 +4,8 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
+# !!!!!!!!
+# IF YOU EDIT THIS FILE, YOU MUST EDIT THE `connectors_utility.gemspec`
 require 'utility/bulk_queue'
 require 'utility/common'
 require 'utility/constants'
@@ -11,9 +13,12 @@ require 'utility/cron'
 require 'utility/elasticsearch/index/mappings'
 require 'utility/elasticsearch/index/text_analysis_settings'
 require 'utility/environment'
+require 'utility/error_monitor'
 require 'utility/errors'
 require 'utility/filtering'
 require 'utility/es_client'
 require 'utility/exception_tracking'
 require 'utility/extension_mapping_util'
 require 'utility/logger'
+# IF YOU EDIT THIS FILE, YOU MUST EDIT THE `connectors_utility.gemspec`
+# !!!!!!!!

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_utility
 version: !ruby/object:Gem::Version
-  version: 8.6.0.4.pre.20221115T002329Z
+  version: 8.7.0.0.pre.20221117T004939Z
 platform: ruby
 authors:
 - Elastic
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-15 00:00:00.000000000 Z
+date: 2022-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -119,6 +119,7 @@ files:
 - lib/utility/elasticsearch/index/mappings.rb
 - lib/utility/elasticsearch/index/text_analysis_settings.rb
 - lib/utility/environment.rb
+- lib/utility/error_monitor.rb
 - lib/utility/errors.rb
 - lib/utility/es_client.rb
 - lib/utility/exception_tracking.rb
@@ -129,9 +130,9 @@ homepage: https://github.com/elastic/connectors-ruby
 licenses:
 - Elastic-2.0
 metadata:
-  revision: f506d5e5ebedfb0c6058d347d8ce22adc42e2cc0
-  repository: git@github.com:elastic/ent-search-connectors.git
-post_install_message: 
+  revision: 294214a26b0fe9a4347763b01de681c336e8daae
+  repository: https://github.com/elastic/connectors-ruby.git
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -147,7 +148,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3.1
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Gem containing shared Connector Services libraries
 test_files: []