connectors_utility 8.6.0.4.pre.20221116T024609Z → 8.6.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ae5315b00ad59db8c9a0300ec34f560a8293a285aa0d62fb5b9f662996b432d
-  data.tar.gz: 6eec049ff3a257bcff893edefd841e9b441dcc1858f7f88c4b0376fe47961bcc
+  metadata.gz: bd3d8a98ffaf8965434e1c01467c4ff30d87a1b25e414d5ce36f8a8529178053
+  data.tar.gz: 07d54a470c31ba311aeee4aca41401bca11178d20ed7de72c8dadf7926e910d4
 SHA512:
-  metadata.gz: 296e79a4f866b91a98d02cf45ab7742efc2e10dd209be9456fcf6be34600247d55b09e7313030a386ccef693207b20b8802d5cb2ded3662cc289ed70fe5c1c3f
-  data.tar.gz: 7d59558a77f22e14a01ef279ae1da29195b1f1216920e6aecde81c2c5b5f11371a5f5fc23b87df5587ca3c1969389d86e5b02c189e24f94ea5a0590047af4b98
+  metadata.gz: eddc02838146053fa61173a5dc6060eb62c92c70930bd5c1338586572af08078288455c65fb2301796d99b338b996ec889966e535bbceaf9f1f39bac6c9888bc
+  data.tar.gz: e0ec181ccc3cb8dd59f123003d873c1218f2e75c6d91569efe075330d913d6bb366c4bcfbc389844b8040a10e22e5b8c797525379d042cb81a2fa2c7e3af08a4

data/lib/core/connector_job.rb

@@ -0,0 +1,240 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'connectors/sync_status'
+require 'core/connector_settings'
+require 'core/elastic_connector_actions'
+require 'utility'
+
+module Core
+  class ConnectorJob
+    DEFAULT_PAGE_SIZE = 100
+    STUCK_THRESHOLD = 60
+
+    def self.fetch_by_id(job_id)
+      es_response = ElasticConnectorActions.get_job(job_id)
+      return nil unless es_response[:found]
+
+      new(es_response)
+    end
+
+    def self.pending_jobs(connectors_ids: [], page_size: DEFAULT_PAGE_SIZE)
+      status_term = { status: Connectors::SyncStatus::PENDING_STATUSES }
+
+      query = { bool: { must: [{ terms: status_term }] } }
+
+      return fetch_jobs_by_query(query, page_size) if connectors_ids.empty?
+
+      query[:bool][:must] << { terms: { 'connector.id' => connectors_ids } }
+
+      fetch_jobs_by_query(query, page_size)
+    end
+
+    def self.orphaned_jobs(page_size = DEFAULT_PAGE_SIZE)
+      connector_ids = ConnectorSettings.fetch_all_connectors.map(&:id)
+      query = { bool: { must_not: { terms: { 'connector.id': connector_ids } } } }
+      fetch_jobs_by_query(query, page_size)
+    end
+
+    def self.delete_jobs(jobs)
+      query = { terms: { '_id': jobs.map(&:id) } }
+      ElasticConnectorActions.delete_jobs_by_query(query)
+    end
+
+    def self.stuck_jobs(connector_id = nil, page_size = DEFAULT_PAGE_SIZE)
+      connector_ids = if connector_id
+                        [connector_id]
+                      else
+                        ConnectorSettings.fetch_native_connectors.map(&:id)
+                      end
+      query = {
+        bool: {
+          filter: [
+              { terms: { 'connector.id': connector_ids } },
+              { terms: { status: Connectors::SyncStatus::ACTIVE_STATUSES } },
+              { range: { last_seen: { lte: "now-#{STUCK_THRESHOLD}s" } } }
+          ]
+        }
+      }
+      fetch_jobs_by_query(query, page_size)
+    end
+
+    def self.enqueue(_connector_id)
+      nil
+    end
+
+    def id
+      @elasticsearch_response[:_id]
+    end
+
+    def [](property_name)
+      @elasticsearch_response[:_source][property_name]
+    end
+
+    def error
+      self[:error]
+    end
+
+    def status
+      self[:status]
+    end
+
+    def in_progress?
+      status == Connectors::SyncStatus::IN_PROGRESS
+    end
+
+    def canceling?
+      status == Connectors::SyncStatus::CANCELING
+    end
+
+    def suspended?
+      status == Connectors::SyncStatus::SUSPENDED
+    end
+
+    def canceled?
+      status == Connectors::SyncStatus::CANCELED
+    end
+
+    def pending?
+      Connectors::SyncStatus::PENDING_STATUSES.include?(status)
+    end
+
+    def active?
+      Connectors::SyncStatus::ACTIVE_STATUSES.include?(status)
+    end
+
+    def terminated?
+      Connectors::SyncStatus::TERMINAL_STATUSES.include?(status)
+    end
+
+    def connector_snapshot
+      self[:connector] || {}
+    end
+
+    def connector_id
+      connector_snapshot[:id]
+    end
+
+    def index_name
+      connector_snapshot[:index_name]
+    end
+
+    def language
+      connector_snapshot[:language]
+    end
+
+    def service_type
+      connector_snapshot[:service_type]
+    end
+
+    def configuration
+      connector_snapshot[:configuration]
+    end
+
+    def filtering
+      connector_snapshot[:filtering]
+    end
+
+    def pipeline
+      connector_snapshot[:pipeline]
+    end
+
+    def connector
+      @connector ||= ConnectorSettings.fetch_by_id(connector_id)
+    end
+
+    def update_metadata(ingestion_stats = {}, connector_metadata = {})
+      ingestion_stats ||= {}
+      doc = { :last_seen => Time.now }.merge(ingestion_stats)
+      doc[:metadata] = connector_metadata if connector_metadata&.any?
+      ElasticConnectorActions.update_job_fields(id, doc)
+    end
+
+    def done!(ingestion_stats = {}, connector_metadata = {})
+      terminate!(Connectors::SyncStatus::COMPLETED, nil, ingestion_stats, connector_metadata)
+    end
+
+    def error!(message, ingestion_stats = {}, connector_metadata = {})
+      terminate!(Connectors::SyncStatus::ERROR, message, ingestion_stats, connector_metadata)
+    end
+
+    def cancel!(ingestion_stats = {}, connector_metadata = {})
+      terminate!(Connectors::SyncStatus::CANCELED, nil, ingestion_stats, connector_metadata)
+    end
+
+    def with_concurrency_control
+      response = ElasticConnectorActions.get_job(id)
+
+      yield response, response['_seq_no'], response['_primary_term']
+    end
+
+    def make_running!
+      with_concurrency_control do |es_doc, seq_no, primary_term|
+        now = Time.now
+        doc = {
+          status: Connectors::SyncStatus::IN_PROGRESS,
+          started_at: now,
+          last_seen: now,
+          worker_hostname: Socket.gethostname
+        }
+
+        ElasticConnectorActions.update_job_fields(es_doc[:_id], doc, seq_no, primary_term)
+      end
+    end
+
+    def es_source
+      @elasticsearch_response[:_source]
+    end
+
+    private
+
+    def self.fetch_jobs_by_query(query, page_size)
+      results = []
+      offset = 0
+      loop do
+        response = ElasticConnectorActions.search_jobs(query, page_size, offset)
+
+        hits = response.dig('hits', 'hits') || []
+        total = response.dig('hits', 'total', 'value') || 0
+        results += hits.map { |hit| new(hit) }
+        break if results.size >= total
+        offset += hits.size
+      end
+
+      results
+    end
+
+    def initialize(es_response)
+      # TODO: remove the usage of with_indifferent_access. The initialize method should expect a hash argument
+      @elasticsearch_response = es_response.with_indifferent_access
+    end
+
+    def terminate!(status, error = nil, ingestion_stats = {}, connector_metadata = {})
+      ingestion_stats ||= {}
+      ingestion_stats[:total_document_count] = ElasticConnectorActions.document_count(index_name)
+      doc = {
+        :last_seen => Time.now,
+        :completed_at => Time.now,
+        :status => status,
+        :error => error
+      }.merge(ingestion_stats)
+      doc[:canceled_at] = Time.now if status == Connectors::SyncStatus::CANCELED
+      doc[:metadata] = connector_metadata if connector_metadata&.any?
+      ElasticConnectorActions.update_job_fields(id, doc)
+    end
+
+    def seq_no
+      @elasticsearch_response[:_seq_no]
+    end
+
+    def primary_term
+      @elasticsearch_response[:_primary_term]
+    end
+  end
+end

data/lib/core/connector_settings.rb

@@ -49,6 +49,11 @@ module Core
       fetch_connectors_by_query(query, page_size)
     end
 
+    def self.fetch_all_connectors(page_size = DEFAULT_PAGE_SIZE)
+      query = { match_all: {} }
+      fetch_connectors_by_query(query, page_size)
+    end
+
     def id
       @elasticsearch_response[:_id]
     end
@@ -130,19 +135,21 @@ module Core
     end
 
     def update_last_sync!(job)
+      # if job is nil, connector still needs to be updated, to avoid it stuck at in_progress
+      job_status = job&.status || Connectors::SyncStatus::ERROR
+      job_error = job.nil? ? 'Could\'t find the job' : job.error
+      job_error ||= 'unknown error' if job_status == Connectors::SyncStatus::ERROR
       doc = {
-        :last_sync_status => job.status,
+        :last_sync_status => job_status,
         :last_synced => Time.now,
-        :last_sync_error => job.error,
-        :error => job.error
+        :last_sync_error => job_error,
+        :error => job_error
       }
-
-      if job.terminated?
+      if job&.terminated?
         doc[:last_indexed_document_count] = job[:indexed_document_count]
         doc[:last_deleted_document_count] = job[:deleted_document_count]
       end
-
-      Core::ElasticConnectorActions.update_connector_fields(job.connector_id, doc)
+      Core::ElasticConnectorActions.update_connector_fields(id, doc)
     end
 
     private

data/lib/core/elastic_connector_actions.rb

@@ -91,6 +91,17 @@ module Core
         )
       end
 
+      def delete_jobs_by_query(query)
+        client.delete_by_query(
+          :index => Utility::Constants::JOB_INDEX,
+          :body => { :query => query }
+        )
+      end
+
+      def delete_indices(indices)
+        client.indices.delete(:index => indices, :ignore_unavailable => true)
+      end
+
       def update_connector_configuration(connector_id, configuration)
         update_connector_fields(connector_id, :configuration => configuration)
       end
@@ -220,37 +231,6 @@ module Core
         update_connector_fields(connector_id, body)
       end
 
-      def update_sync(job_id, metadata)
-        body = {
-          :doc => { :last_seen => Time.now }.merge(metadata)
-        }
-        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
-      end
-
-      def complete_sync(connector_id, job_id, metadata, error)
-        sync_status = error ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
-
-        metadata ||= {}
-
-        update_connector_fields(connector_id,
-                                :last_sync_status => sync_status,
-                                :last_sync_error => error,
-                                :error => error,
-                                :last_synced => Time.now,
-                                :last_indexed_document_count => metadata[:indexed_document_count],
-                                :last_deleted_document_count => metadata[:deleted_document_count])
-
-        body = {
-          :doc => {
-            :status => sync_status,
-            :completed_at => Time.now,
-            :last_seen => Time.now,
-            :error => error
-          }.merge(metadata)
-        }
-        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
-      end
-
       def fetch_document_ids(index_name)
         page_size = 1000
         result = []

data/lib/core/scheduler.rb

@@ -90,13 +90,6 @@ module Core
         return false
       end
 
-      # We want to sync when sync never actually happened
-      last_synced = connector_settings[:last_synced]
-      if last_synced.nil? || last_synced.empty?
-        Utility::Logger.info("#{connector_settings.formatted.capitalize} has never synced yet, running initial sync.")
-        return true
-      end
-
       current_schedule = scheduling_settings[:interval]
 
       # Don't sync if there is no actual scheduling interval
@@ -119,6 +112,13 @@ module Core
         return false
       end
 
+      # We want to sync when sync never actually happened
+      last_synced = connector_settings[:last_synced]
+      if last_synced.nil? || last_synced.empty?
+        Utility::Logger.info("#{connector_settings.formatted.capitalize} has never synced yet, running initial sync.")
+        return true
+      end
+
       next_trigger_time = cron_parser.next_time(Time.parse(last_synced))
 
       # Sync if next trigger for the connector is in past

data/lib/utility/bulk_queue.rb

@@ -6,12 +6,14 @@
 
 require 'json'
 
+require 'utility/constants'
+
 module Utility
   class BulkQueue
     class QueueOverflowError < StandardError; end
 
     # 500 items or 5MB
-    def initialize(operation_count_threshold = 500, size_threshold = 5 * 1024 * 1024)
+    def initialize(operation_count_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_SIZE, size_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES)
       @operation_count_threshold = operation_count_threshold.freeze
       @size_threshold = size_threshold.freeze
 

data/lib/utility/constants.rb

@@ -18,5 +18,10 @@ module Utility
     CRAWLER_SERVICE_TYPE = 'elastic-crawler'
     FILTERING_RULES_FEATURE = 'filtering_rules'
     FILTERING_ADVANCED_FEATURE = 'filtering_advanced_config'
+
+    # Maximum number of operations in BULK Elasticsearch operation that will ingest the data
+    DEFAULT_MAX_INGESTION_QUEUE_SIZE = 500
+    # Maximum size of either whole BULK Elasticsearch operation or one document in it
+    DEFAULT_MAX_INGESTION_QUEUE_BYTES = 5 * 1024 * 1024
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_utility
 version: !ruby/object:Gem::Version
-  version: 8.6.0.4.pre.20221116T024609Z
+  version: 8.6.0.5
 platform: ruby
 authors:
 - Elastic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-16 00:00:00.000000000 Z
+date: 2022-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -106,6 +106,7 @@ files:
 - lib/connectors/crawler/scheduler.rb
 - lib/connectors/sync_status.rb
 - lib/connectors_utility.rb
+- lib/core/connector_job.rb
 - lib/core/connector_settings.rb
 - lib/core/elastic_connector_actions.rb
 - lib/core/filtering/validation_status.rb
@@ -130,8 +131,8 @@ homepage: https://github.com/elastic/connectors-ruby
 licenses:
 - Elastic-2.0
 metadata:
-  revision: b3cc1332879a38930a272a63f8c6be1847578204
-  repository: git@github.com:elastic/ent-search-connectors.git
+  revision: 39cbb85dbae57a2c92e6e0da272d05aa24ca99a9
+  repository: git@github.com:elastic/connectors-ruby.git
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -143,9 +144,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.0.3.1
 signing_key: