connectors_service 8.6.0.4.pre.20221116T024501Z → 8.6.0.4

data/lib/core/elastic_connector_actions.rb

@@ -19,12 +19,6 @@ module Core
     end
   end
 
-  class JobNotCreatedError < StandardError
-    def initialize(connector_id, response)
-      super("Sync job for connector '#{connector_id}' could not be created. Response: #{response}")
-    end
-  end
-
   class ConnectorVersionChangedError < StandardError
     def initialize(connector_id, seq_no, primary_term)
       super("Version conflict: seq_no [#{seq_no}] and primary_term [#{primary_term}] do not match for connector '#{connector_id}'.")
@@ -49,17 +43,10 @@ module Core
       end
 
       def get_connector(connector_id)
-        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
         client.get(:index => Utility::Constants::CONNECTORS_INDEX, :id => connector_id, :ignore => 404).with_indifferent_access
       end
 
-      def get_job(job_id)
-        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
-        client.get(:index => Utility::Constants::JOB_INDEX, :id => job_id, :ignore => 404).with_indifferent_access
-      end
-
       def connectors_meta
-        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
         alias_mappings = client.indices.get_mapping(:index => Utility::Constants::CONNECTORS_INDEX).with_indifferent_access
         index = get_latest_index_in_alias(Utility::Constants::CONNECTORS_INDEX, alias_mappings.keys)
         alias_mappings.dig(index, 'mappings', '_meta') || {}
@@ -78,19 +65,6 @@ module Core
         )
       end
 
-      def search_jobs(query, page_size, offset)
-        client.search(
-          :index => Utility::Constants::JOB_INDEX,
-          :ignore => 404,
-          :body => {
-              :size => page_size,
-              :from => offset,
-              :query => query,
-              :sort => ['created_at']
-          }
-        )
-      end
-
       def update_connector_configuration(connector_id, configuration)
         update_connector_fields(connector_id, :configuration => configuration)
       end
@@ -110,57 +84,11 @@ module Core
         update_connector_configuration(connector_id, payload)
       end
 
-      def update_filtering_validation(connector_id, filter_validation_results)
-        return if filter_validation_results.empty?
-
-        filtering = get_connector(connector_id).dig(:_source, :filtering)
-
-        case filtering
-        when Hash
-          update_filter_validation(filtering, filter_validation_results)
-        when Array
-          return unless should_update_validations?(filter_validation_results, filtering)
-
-          filtering.each do |filter|
-            update_filter_validation(filter, filter_validation_results)
-          end
-        else
-          Utility::Logger.warn("Elasticsearch returned invalid filtering format: #{filtering}. Skipping validation.")
-          return
-        end
-
-        update_connector_fields(connector_id, { :filtering => filtering })
-      end
-
-      def update_connector_sync_now(connector_id, sync_now)
-        doc = connector_with_concurrency_control(connector_id)
-
-        body = { sync_now: sync_now, last_synced: Time.now }
-
-        update_connector_fields(
-          connector_id,
-          body,
-          doc[:seq_no],
-          doc[:primary_term]
-        )
-      end
-
-      def update_connector_last_sync_status(connector_id, last_sync_status)
-        doc = connector_with_concurrency_control(connector_id)
-
-        update_connector_fields(
-          connector_id,
-          { last_sync_status: last_sync_status },
-          doc[:seq_no],
-          doc[:primary_term]
-        )
-      end
-
-      def connector_with_concurrency_control(connector_id)
+      def claim_job(connector_id)
         seq_no = nil
         primary_term = nil
-
-        doc = client.get(
+        sync_in_progress = false
+        connector_record = client.get(
           :index => Utility::Constants::CONNECTORS_INDEX,
           :id => connector_id,
           :ignore => 404,
@@ -168,42 +96,39 @@ module Core
         ).tap do |response|
           seq_no = response['_seq_no']
           primary_term = response['_primary_term']
+          sync_in_progress = response.dig('_source', 'last_sync_status') == Connectors::SyncStatus::IN_PROGRESS
         end
+        if sync_in_progress
+          raise JobAlreadyRunningError.new(connector_id)
+        end
+        update_connector_fields(
+          connector_id,
+          { :sync_now => false,
+            :last_sync_status => Connectors::SyncStatus::IN_PROGRESS,
+            :last_synced => Time.now },
+          seq_no,
+          primary_term
+        )
 
-        { doc: doc, seq_no: seq_no, primary_term: primary_term }
-      end
-
-      def create_job(connector_settings:)
         body = {
-          status: Connectors::SyncStatus::PENDING,
-          created_at: Time.now,
-          last_seen: Time.now,
-          connector: {
-            id: connector_settings.id,
-            filtering: convert_connector_filtering_to_job_filtering(connector_settings.filtering),
-            index_name: connector_settings.index_name,
-            language: connector_settings[:language],
-            pipeline: connector_settings[:pipeline],
-            service_type: connector_settings.service_type
-          }
+          :connector_id => connector_id,
+          :status => Connectors::SyncStatus::IN_PROGRESS,
+          :worker_hostname => Socket.gethostname,
+          :created_at => Time.now,
+          :filtering => convert_connector_filtering_to_job_filtering(connector_record.dig('_source', 'filtering'))
         }
 
-        index_response = client.index(index: Utility::Constants::JOB_INDEX, body: body, refresh: true)
-
-        return index_response if index_response['result'] == 'created'
-
-        raise JobNotCreatedError.new(connector_settings.id, index_response)
+        client.index(:index => Utility::Constants::JOB_INDEX, :body => body)
       end
 
       def convert_connector_filtering_to_job_filtering(connector_filtering)
         return [] unless connector_filtering
         connector_filtering = [connector_filtering] unless connector_filtering.is_a?(Array)
         connector_filtering.each_with_object([]) do |filtering_domain, job_filtering|
-          snippet = filtering_domain.dig('active', 'advanced_snippet') || {}
           job_filtering << {
             'domain' => filtering_domain['domain'],
             'rules' => filtering_domain.dig('active', 'rules'),
-            'advanced_snippet' => snippet['value'] || snippet,
+            'advanced_snippet' => filtering_domain.dig('active', 'advanced_snippet'),
             'warnings' => [] # TODO: in https://github.com/elastic/enterprise-search-team/issues/3174
           }
         end
@@ -220,33 +145,22 @@ module Core
         update_connector_fields(connector_id, body)
       end
 
-      def update_sync(job_id, metadata)
-        body = {
-          :doc => { :last_seen => Time.now }.merge(metadata)
-        }
-        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
-      end
-
-      def complete_sync(connector_id, job_id, metadata, error)
-        sync_status = error ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
-
-        metadata ||= {}
+      def complete_sync(connector_id, job_id, status)
+        sync_status = status[:error] ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
 
         update_connector_fields(connector_id,
                                 :last_sync_status => sync_status,
-                                :last_sync_error => error,
-                                :error => error,
+                                :last_sync_error => status[:error],
+                                :error => status[:error],
                                 :last_synced => Time.now,
-                                :last_indexed_document_count => metadata[:indexed_document_count],
-                                :last_deleted_document_count => metadata[:deleted_document_count])
+                                :last_indexed_document_count => status[:indexed_document_count],
+                                :last_deleted_document_count => status[:deleted_document_count])
 
         body = {
           :doc => {
             :status => sync_status,
-            :completed_at => Time.now,
-            :last_seen => Time.now,
-            :error => error
-          }.merge(metadata)
+            :completed_at => Time.now
+          }.merge(status)
         }
         client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
       end
@@ -334,105 +248,12 @@ module Core
           :properties => {
             :api_key_id => { :type => :keyword },
             :configuration => { :type => :object },
-            :description => { :type => :text },
-            :error => { :type => :keyword },
-            :features => {
-              :properties => {
-                :filtering_advanced_config => { :type => :boolean },
-                :filtering_rules => { :type => :boolean }
-              }
-            },
-            :filtering => {
-              :properties => {
-                :domain => { :type => :keyword },
-                :active => {
-                  :properties => {
-                    :rules => {
-                      :properties => {
-                        :id => { :type => :keyword },
-                        :policy => { :type => :keyword },
-                        :field => { :type => :keyword },
-                        :rule => { :type => :keyword },
-                        :value => { :type => :keyword },
-                        :order => { :type => :short },
-                        :created_at => { :type => :date },
-                        :updated_at => { :type => :date }
-                      }
-                    },
-                    :advanced_snippet => {
-                      :properties => {
-                        :value => { :type => :object },
-                        :created_at => { :type => :date },
-                        :updated_at => { :type => :date }
-                       }
-                    },
-                    :validation => {
-                      :properties => {
-                        :state => { :type => :keyword },
-                        :errors => {
-                          :properties => {
-                            :ids => { :type => :keyword },
-                            :messages => { :type => :text }
-                          }
-                        }
-                      }
-                    }
-                  }
-                },
-                :draft => {
-                  :properties => {
-                    :rules => {
-                      :properties => {
-                        :id => { :type => :keyword },
-                        :policy => { :type => :keyword },
-                        :field => { :type => :keyword },
-                        :rule => { :type => :keyword },
-                        :value => { :type => :keyword },
-                        :order => { :type => :short },
-                        :created_at => { :type => :date },
-                        :updated_at => { :type => :date }
-                      }
-                    },
-                    :advanced_snippet => {
-                      :properties => {
-                        :value => { :type => :object },
-                        :created_at => { :type => :date },
-                        :updated_at => { :type => :date }
-                      }
-                    },
-                    :validation => {
-                      :properties => {
-                        :state => { :type => :keyword },
-                        :errors => {
-                          :properties => {
-                            :ids => { :type => :keyword },
-                            :messages => { :type => :text }
-                          }
-                        }
-                      }
-                    }
-                  }
-                }
-              }
-            },
+            :error => { :type => :text },
             :index_name => { :type => :keyword },
-            :is_native => { :type => :boolean },
-            :language => { :type => :keyword },
             :last_seen => { :type => :date },
-            :last_sync_error => { :type => :keyword },
-            :last_sync_status => { :type => :keyword },
             :last_synced => { :type => :date },
-            :last_deleted_document_count => { :type => :long },
-            :last_indexed_document_count => { :type => :long },
-            :name => { :type => :keyword },
-            :pipeline => {
-              :properties => {
-                :extract_binary_content => { :type => :boolean },
-                :name => { :type => :keyword },
-                :reduce_whitespace => { :type => :boolean },
-                :run_ml_inference => { :type => :boolean }
-              }
-            },
+            :last_indexed_document_count => { :type => :integer },
+            :last_deleted_document_count => { :type => :integer },
             :scheduling => {
               :properties => {
                 :enabled => { :type => :boolean },
@@ -441,7 +262,9 @@ module Core
             },
             :service_type => { :type => :keyword },
             :status => { :type => :keyword },
-            :sync_now => { :type => :boolean }
+            :sync_error => { :type => :text },
+            :sync_now => { :type => :boolean },
+            :sync_status => { :type => :keyword }
           }
         }
         ensure_index_exists("#{Utility::Constants::CONNECTORS_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::CONNECTORS_INDEX, :mappings => mappings))
@@ -452,139 +275,55 @@ module Core
       def ensure_job_index_exists
         mappings = {
           :properties => {
-            :cancelation_requested_at => { :type => :date },
-            :canceled_at => { :type => :date },
-            :completed_at => { :type => :date },
-            :connector => {
-              :properties => {
-                :configuration => { :type => :object },
-                :filtering => {
-                  :properties => {
-                    :domain => { :type => :keyword },
-                    :rules => {
-                      :properties => {
-                        :id => { :type => :keyword },
-                        :policy => { :type => :keyword },
-                        :field => { :type => :keyword },
-                        :rule => { :type => :keyword },
-                        :value => { :type => :keyword },
-                        :order => { :type => :short },
-                        :created_at => { :type => :date },
-                        :updated_at => { :type => :date }
-                      }
-                    },
-                    :advanced_snippet => {
-                      :properties => {
-                        :value => { :type => :object },
-                        :created_at => { :type => :date },
-                        :updated_at => { :type => :date }
-                      }
-                    },
-                    :warnings => {
-                      :properties => {
-                        :ids => { :type => :keyword },
-                        :messages => { :type => :text }
-                      }
-                    }
-                  }
-                },
-                :id => { :type => :keyword },
-                :index_name => { :type => :keyword },
-                :language => { :type => :keyword },
-                :pipeline => {
-                  :properties => {
-                    :extract_binary_content => { :type => :boolean },
-                    :name => { :type => :keyword },
-                    :reduce_whitespace => { :type => :boolean },
-                    :run_ml_inference => { :type => :boolean }
-                  }
-                },
-                :service_type => { :type => :keyword }
-              }
-            },
-            :created_at => { :type => :date },
-            :deleted_document_count => { :type => :integer },
+            :connector_id => { :type => :keyword },
+            :status => { :type => :keyword },
             :error => { :type => :text },
+            :worker_hostname => { :type => :keyword },
             :indexed_document_count => { :type => :integer },
-            :indexed_document_volume => { :type => :integer },
-            :last_seen => { :type => :date },
-            :metadata => { :type => :object },
-            :started_at => { :type => :date },
-            :status => { :type => :keyword },
-            :total_document_count => { :type => :integer },
-            :trigger_method => { :type => :keyword },
-            :worker_hostname => { :type => :keyword }
+            :deleted_document_count => { :type => :integer },
+            :created_at => { :type => :date },
+            :completed_at => { :type => :date }
           }
         }
         ensure_index_exists("#{Utility::Constants::JOB_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::JOB_INDEX, :mappings => mappings))
       end
 
       def update_connector_fields(connector_id, doc = {}, seq_no = nil, primary_term = nil)
-        update_doc_fields(Utility::Constants::CONNECTORS_INDEX, connector_id, doc, seq_no, primary_term)
-      end
-
-      def update_job_fields(job_id, doc = {}, seq_no = nil, primary_term = nil)
-        update_doc_fields(Utility::Constants::JOB_INDEX, job_id, doc, seq_no, primary_term)
-      end
-
-      def document_count(index_name)
-        client.indices.refresh(:index => index_name)
-        client.count(:index => index_name)['count']
-      end
-
-      private
-
-      def should_update_validations?(domain_validations, filtering)
-        domains_present = filtering.collect { |filter| filter[:domain] }
-        domains_to_update = domain_validations.keys
-
-        # non-empty intersection -> domains to update present
-        !(domains_present & domains_to_update).empty?
-      end
-
-      def client
-        @client ||= Utility::EsClient.new(App::Config[:elasticsearch])
-      end
-
-      def get_latest_index_in_alias(alias_name, indicies)
-        index_versions = indicies.map { |index| index.gsub("#{alias_name}-v", '').to_i }
-        index_version = index_versions.max # gets the largest suffix number
-        "#{alias_name}-v#{index_version}"
-      end
-
-      def update_filter_validation(filter, domain_validations)
-        domain = filter[:domain]
-
-        if domain_validations.key?(domain)
-          new_validation_state = { :draft => { :validation => domain_validations[domain] } }
-          filter.deep_merge!(new_validation_state)
-        end
-      end
-
-      def update_doc_fields(index, id, doc = {}, seq_no = nil, primary_term = nil)
         return if doc.empty?
         update_args = {
-          :index => index,
-          :id => id,
+          :index => Utility::Constants::CONNECTORS_INDEX,
+          :id => connector_id,
           :body => { :doc => doc },
           :refresh => true,
           :retry_on_conflict => 3
         }
-
+        # seq_no and primary_term are used for optimistic concurrency control
+        # see https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html
         if seq_no && primary_term
           update_args[:if_seq_no] = seq_no
           update_args[:if_primary_term] = primary_term
           update_args.delete(:retry_on_conflict)
         end
-
         begin
           client.update(update_args)
         rescue Elastic::Transport::Transport::Errors::Conflict
           # VersionConflictException
           # see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#optimistic-concurrency-control-index
-          raise ConnectorVersionChangedError.new(id, seq_no, primary_term)
+          raise ConnectorVersionChangedError.new(connector_id, seq_no, primary_term)
         end
       end
+
+      private
+
+      def client
+        @client ||= Utility::EsClient.new(App::Config[:elasticsearch])
+      end
+
+      def get_latest_index_in_alias(alias_name, indicies)
+        index_versions = indicies.map { |index| index.gsub("#{alias_name}-v", '').to_i }
+        index_version = index_versions.max # gets the largest suffix number
+        "#{alias_name}-v#{index_version}"
+      end
     end
   end
 end

data/lib/core/output_sink/base_sink.rb

@@ -0,0 +1,33 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Core
+  module OutputSink
+    class BaseSink
+      def ingest(_document)
+        raise 'not implemented'
+      end
+
+      def ingest_multiple(_documents)
+        raise 'not implemented'
+      end
+
+      def delete(_id)
+        raise 'not implemented'
+      end
+
+      def delete_multiple(_ids)
+        raise 'not implemented'
+      end
+
+      def flush(_size: nil)
+        raise 'not implemented'
+      end
+    end
+  end
+end

data/lib/core/output_sink/combined_sink.rb

@@ -0,0 +1,38 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/output_sink/base_sink'
+require 'utility/logger'
+
+module Core::OutputSink
+  class CombinedSink < Core::OutputSink::BaseSink
+    def initialize(sinks = [])
+      @sinks = sinks
+    end
+
+    def ingest(document)
+      @sinks.each { |sink| sink.ingest(document) }
+    end
+
+    def flush(size: nil)
+      @sinks.each { |sink| sink.flush(size: size) }
+    end
+
+    def ingest_multiple(documents)
+      @sinks.each { |sink| sink.ingest_multiple(documents) }
+    end
+
+    def delete(id)
+      @sinks.each { |sink| sink.delete(id) }
+    end
+
+    def delete_multiple(ids)
+      @sinks.each { |sink| sink.delete_multiple(ids) }
+    end
+  end
+end

data/lib/core/output_sink/console_sink.rb

@@ -0,0 +1,51 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/output_sink'
+require 'utility/logger'
+
+module Core::OutputSink
+  class ConsoleSink < Core::OutputSink::BaseSink
+    def ingest(document)
+      print_header 'Got a single document:'
+      puts document
+    end
+
+    def flush(size: nil)
+      print_header 'Flushing'
+      puts "Flush size: #{size}"
+    end
+
+    def ingest_multiple(documents)
+      print_header 'Got multiple documents:'
+      puts documents
+    end
+
+    def delete(id)
+      print_header "Deleting single id: #{id}"
+      puts id
+    end
+
+    def delete_multiple(ids)
+      print_header "Deleting several ids: #{ids}"
+      puts ids
+    end
+
+    private
+
+    def print_delim
+      puts '----------------------------------------------------'
+    end
+
+    def print_header(header)
+      print_delim
+      puts header
+      print_delim
+    end
+  end
+end

data/lib/core/output_sink/es_sink.rb

@@ -0,0 +1,74 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/numeric/time'
+require 'app/config'
+require 'core/output_sink/base_sink'
+require 'utility/es_client'
+require 'utility/logger'
+
+module Core::OutputSink
+  class EsSink < Core::OutputSink::BaseSink
+    def initialize(index_name, request_pipeline, flush_threshold = 50)
+      super()
+      @client = Utility::EsClient.new(App::Config[:elasticsearch])
+      @index_name = index_name
+      @request_pipeline = request_pipeline
+      @operation_queue = []
+      @flush_threshold = flush_threshold
+    end
+
+    def ingest(document)
+      return if document.blank?
+
+      @operation_queue << { :index => { :_index => index_name, :_id => document[:id], :data => document } }
+      flush if ready_to_flush?
+    end
+
+    def delete(doc_id)
+      return if doc_id.nil?
+
+      @operation_queue << { :delete => { :_index => index_name, :_id => doc_id } }
+      flush if ready_to_flush?
+    end
+
+    def flush(size: nil)
+      flush_size = size || @flush_threshold
+
+      while @operation_queue.any?
+        data_to_flush = @operation_queue.pop(flush_size)
+        send_data(data_to_flush)
+      end
+    end
+
+    def ingest_multiple(documents)
+      Utility::Logger.debug "Enqueueing #{documents&.size} documents to the index #{index_name}."
+      documents.each { |doc| ingest(doc) }
+    end
+
+    def delete_multiple(ids)
+      Utility::Logger.debug "Enqueueing #{ids&.size} ids to delete from the index #{index_name}."
+      ids.each { |id| delete(id) }
+    end
+
+    private
+
+    attr_accessor :index_name
+
+    def send_data(ops)
+      return if ops.empty?
+
+      @client.bulk(:body => ops, :pipeline => @request_pipeline)
+      Utility::Logger.info "Applied #{ops.size} upsert/delete operations to the index #{index_name}."
+    end
+
+    def ready_to_flush?
+      @operation_queue.size >= @flush_threshold
+    end
+  end
+end