connectors_service 8.6.0.4 → 8.7.0.0.pre.20221117T010623Z

data/lib/core/connector_job.rb

@@ -0,0 +1,210 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'connectors/sync_status'
+require 'core/elastic_connector_actions'
+require 'utility'
+
+module Core
+  class ConnectorJob
+    DEFAULT_PAGE_SIZE = 100
+
+    def self.fetch_by_id(job_id)
+      es_response = ElasticConnectorActions.get_job(job_id)
+      return nil unless es_response[:found]
+
+      new(es_response)
+    end
+
+    def self.pending_jobs(connectors_ids: [], page_size: DEFAULT_PAGE_SIZE)
+      status_term = { status: Connectors::SyncStatus::PENDING_STATUSES }
+
+      query = { bool: { must: [{ terms: status_term }] } }
+
+      return fetch_jobs_by_query(query, page_size) if connectors_ids.empty?
+
+      query[:bool][:must] << { terms: { 'connector.id' => connectors_ids } }
+
+      fetch_jobs_by_query(query, page_size)
+    end
+
+    def self.orphaned_jobs(_page_size = DEFAULT_PAGE_SIZE)
+      []
+    end
+
+    def self.stuck_jobs(_page_size = DEFAULT_PAGE_SIZE)
+      []
+    end
+
+    def self.enqueue(_connector_id)
+      nil
+    end
+
+    def id
+      @elasticsearch_response[:_id]
+    end
+
+    def [](property_name)
+      @elasticsearch_response[:_source][property_name]
+    end
+
+    def error
+      self[:error]
+    end
+
+    def status
+      self[:status]
+    end
+
+    def in_progress?
+      status == Connectors::SyncStatus::IN_PROGRESS
+    end
+
+    def canceling?
+      status == Connectors::SyncStatus::CANCELING
+    end
+
+    def suspended?
+      status == Connectors::SyncStatus::SUSPENDED
+    end
+
+    def canceled?
+      status == Connectors::SyncStatus::CANCELED
+    end
+
+    def pending?
+      Connectors::SyncStatus::PENDING_STATUSES.include?(status)
+    end
+
+    def active?
+      Connectors::SyncStatus::ACTIVE_STATUSES.include?(status)
+    end
+
+    def terminated?
+      Connectors::SyncStatus::TERMINAL_STATUSES.include?(status)
+    end
+
+    def connector_snapshot
+      self[:connector] || {}
+    end
+
+    def connector_id
+      @elasticsearch_response[:_source][:connector][:id]
+    end
+
+    def index_name
+      connector_snapshot[:index_name]
+    end
+
+    def language
+      connector_snapshot[:language]
+    end
+
+    def service_type
+      connector_snapshot[:service_type]
+    end
+
+    def configuration
+      connector_snapshot[:configuration]
+    end
+
+    def filtering
+      Utility::Filtering.extract_filter(connector_snapshot[:filtering])
+    end
+
+    def pipeline
+      @elasticsearch_response[:_source][:pipeline]
+    end
+
+    def connector
+      @connector ||= ConnectorSettings.fetch_by_id(connector_id)
+    end
+
+    def done!(ingestion_stats = {}, connector_metadata = {})
+      terminate!(Connectors::SyncStatus::COMPLETED, nil, ingestion_stats, connector_metadata)
+    end
+
+    def error!(message, ingestion_stats = {}, connector_metadata = {})
+      terminate!(Connectors::SyncStatus::ERROR, message, ingestion_stats, connector_metadata)
+    end
+
+    def cancel!(ingestion_stats = {}, connector_metadata = {})
+      terminate!(Connectors::SyncStatus::CANCELED, nil, ingestion_stats, connector_metadata)
+    end
+
+    def with_concurrency_control
+      response = ElasticConnectorActions.get_job(id)
+
+      yield response, response['_seq_no'], response['_primary_term']
+    end
+
+    def make_running!
+      with_concurrency_control do |es_doc, seq_no, primary_term|
+        now = Time.now
+        doc = {
+          status: Connectors::SyncStatus::IN_PROGRESS,
+          started_at: now,
+          last_seen: now,
+          worker_hostname: Socket.gethostname
+        }
+
+        ElasticConnectorActions.update_job_fields(es_doc[:_id], doc, seq_no, primary_term)
+      end
+    end
+
+    def es_source
+      @elasticsearch_response[:_source]
+    end
+
+    private
+
+    def self.fetch_jobs_by_query(query, page_size)
+      results = []
+      offset = 0
+      loop do
+        response = ElasticConnectorActions.search_jobs(query, page_size, offset)
+
+        hits = response.dig('hits', 'hits') || []
+        total = response.dig('hits', 'total', 'value') || 0
+        results += hits.map { |hit| new(hit) }
+        break if results.size >= total
+        offset += hits.size
+      end
+
+      results
+    end
+
+    def initialize(es_response)
+      # TODO: remove the usage of with_indifferent_access. The initialize method should expect a hash argument
+      @elasticsearch_response = es_response.with_indifferent_access
+    end
+
+    def terminate!(status, error = nil, ingestion_stats = {}, connector_metadata = {})
+      ingestion_stats ||= {}
+      ingestion_stats[:total_document_count] = ElasticConnectorActions.document_count(index_name)
+      doc = {
+        :last_seen => Time.now,
+        :completed_at => Time.now,
+        :status => status,
+        :error => error
+      }.merge(ingestion_stats)
+      doc[:canceled_at] = Time.now if status == Connectors::SyncStatus::CANCELED
+      doc[:metadata] = connector_metadata if connector_metadata&.any?
+      ElasticConnectorActions.update_job_fields(id, doc)
+    end
+
+    def seq_no
+      @elasticsearch_response[:_seq_no]
+    end
+
+    def primary_term
+      @elasticsearch_response[:_primary_term]
+    end
+  end
+end

data/lib/core/connector_settings.rb

@@ -23,24 +23,24 @@ module Core
 
     DEFAULT_PAGE_SIZE = 100
 
-    # Error Classes
-    class ConnectorNotFoundError < StandardError; end
-
     def self.fetch_by_id(connector_id)
       es_response = ElasticConnectorActions.get_connector(connector_id)
-      connectors_meta = ElasticConnectorActions.connectors_meta
+      return nil unless es_response[:found]
 
-      raise ConnectorNotFoundError.new("Connector with id=#{connector_id} was not found.") unless es_response[:found]
+      connectors_meta = ElasticConnectorActions.connectors_meta
       new(es_response, connectors_meta)
     end
 
-    def initialize(es_response, connectors_meta)
-      @elasticsearch_response = es_response.with_indifferent_access
-      @connectors_meta = connectors_meta.with_indifferent_access
-    end
-
     def self.fetch_native_connectors(page_size = DEFAULT_PAGE_SIZE)
-      query = { term: { is_native: true } }
+      require 'connectors/registry' unless defined?(Connectors::REGISTRY)
+      query = {
+        bool: {
+          filter: [
+            { term: { is_native: true } },
+            { terms: { service_type: Connectors::REGISTRY.registered_connectors } }
+          ]
+        }
+      }
       fetch_connectors_by_query(query, page_size)
     end
 
@@ -83,23 +83,26 @@ module Core
     end
 
     def filtering
-      Utility::Common.return_if_present(@elasticsearch_response[:filtering], DEFAULT_FILTERING)
+      # assume for now, that first object in filtering array or a filter object itself is the only filtering object
+      filtering = @elasticsearch_response.dig(:_source, :filtering)
+
+      Utility::Filtering.extract_filter(filtering)
     end
 
     def request_pipeline
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
     end
 
     def extract_binary_content?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
     end
 
     def reduce_whitespace?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
     end
 
     def run_ml_inference?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
     end
 
     def formatted
@@ -116,6 +119,39 @@ module Core
       index_name&.start_with?(Utility::Constants::CONTENT_INDEX_PREFIX)
     end
 
+    def ready_for_sync?
+      Connectors::REGISTRY.registered?(service_type) &&
+        valid_index_name? &&
+        connector_status_allows_sync?
+    end
+
+    def running?
+      @elasticsearch_response[:_source][:last_sync_status] == Connectors::SyncStatus::IN_PROGRESS
+    end
+
+    def update_last_sync!(job)
+      doc = {
+        :last_sync_status => job.status,
+        :last_synced => Time.now,
+        :last_sync_error => job.error,
+        :error => job.error
+      }
+
+      if job.terminated?
+        doc[:last_indexed_document_count] = job[:indexed_document_count]
+        doc[:last_deleted_document_count] = job[:deleted_document_count]
+      end
+
+      Core::ElasticConnectorActions.update_connector_fields(job.connector_id, doc)
+    end
+
+    private
+
+    def initialize(es_response, connectors_meta)
+      @elasticsearch_response = es_response.with_indifferent_access
+      @connectors_meta = connectors_meta.with_indifferent_access
+    end
+
     def self.fetch_connectors_by_query(query, page_size)
       connectors_meta = ElasticConnectorActions.connectors_meta