connectors_utility 8.6.0.7 → 8.7.0.0.pre.20221117T004939Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2406276300699ebf07cc2890f994c3a1a9fecb9127793bcad68fb69a4f49111
-  data.tar.gz: 0a35b249367837148c2f7fe2eac30f74de37c5fd5610f30859c703bfbcbb76be
+  metadata.gz: 5d2972f8e6974a79b6088ce6c03453c327132ce19ffb09dbf30f349eae4c2108
+  data.tar.gz: 4fd458de07be07923e0675dc0f341b8211ba3daeec8ac27bfb4f9eb9aff2334a
 SHA512:
-  metadata.gz: e0cca6acc1faffd3312213d807e77425ed006f44313c3b49faf01c8f3b31817ef7a194249581d6058cc8309773fa1142f5daf8994a2a8e24533f1b3f08f73efd
-  data.tar.gz: '078ebf6f05133d0b3ffe252157e72bcfaee89fb98e65d8ffde1a400c69a95f1ea4edfd7dfd66a626955aa24a0e29217cb6f51ead3e30c1c1d2792d52c7ffbb33'
+  metadata.gz: 9db02a3003d5645cbb5d57d4ca1bdb1acb65234f5a931afd9ccb06e2fbbe25be2394c65a72a9ae36038c8c127c35a0d937c83558ede0e4960fc688b073db052a
+  data.tar.gz: d02681e0d4009420b949ec649c9eac52bd533eee493c181ea1ffb15d939b561e81f234bac4885eefd0ea82c8564d7d330c0635c535b0616cfb9280e2e38512df

data/lib/connectors_utility.rb

@@ -9,11 +9,8 @@
 require_relative 'utility'
 
 require_relative 'connectors/connector_status'
-require_relative 'connectors/crawler/scheduler'
-require_relative 'connectors/job_trigger_method'
 require_relative 'connectors/sync_status'
-require_relative 'core/connector_job'
-require_relative 'core/connector_settings'
-require_relative 'core/elastic_connector_actions'
-require_relative 'core/filtering/validation_status'
 require_relative 'core/scheduler'
+require_relative 'core/elastic_connector_actions'
+
+require_relative 'connectors/crawler/scheduler'

data/lib/core/connector_settings.rb

@@ -8,7 +8,6 @@
 
 require 'active_support/core_ext/hash/indifferent_access'
 require 'connectors/connector_status'
-require 'connectors/sync_status'
 require 'core/elastic_connector_actions'
 require 'utility'
 
@@ -50,11 +49,6 @@ module Core
       fetch_connectors_by_query(query, page_size)
     end
 
-    def self.fetch_all_connectors(page_size = DEFAULT_PAGE_SIZE)
-      query = { match_all: {} }
-      fetch_connectors_by_query(query, page_size)
-    end
-
     def id
       @elasticsearch_response[:_id]
     end
@@ -88,10 +82,6 @@ module Core
       self[:scheduling]
     end
 
-    def sync_now?
-      self[:sync_now] == true
-    end
-
     def filtering
       # assume for now, that first object in filtering array or a filter object itself is the only filtering object
       filtering = @elasticsearch_response.dig(:_source, :filtering)
@@ -103,6 +93,18 @@ module Core
       Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
     end
 
+    def extract_binary_content?
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
+    end
+
+    def reduce_whitespace?
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
+    end
+
+    def run_ml_inference?
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
+    end
+
     def formatted
       properties = ["ID: #{id}"]
       properties << "Service type: #{service_type}" if service_type
@@ -128,23 +130,19 @@ module Core
     end
 
     def update_last_sync!(job)
-      # if job is nil, connector still needs to be updated, to avoid it stuck at in_progress
-      job_status = job&.status || Connectors::SyncStatus::ERROR
-      job_error = job.nil? ? 'Could\'t find the job' : job.error
-      job_error ||= 'unknown error' if job_status == Connectors::SyncStatus::ERROR
-      connector_status = (job_status == Connectors::SyncStatus::ERROR ? Connectors::ConnectorStatus::ERROR : Connectors::ConnectorStatus::CONNECTED)
       doc = {
-        :last_sync_status => job_status,
+        :last_sync_status => job.status,
         :last_synced => Time.now,
-        :last_sync_error => job_error,
-        :status => connector_status,
-        :error => job_error
+        :last_sync_error => job.error,
+        :error => job.error
       }
-      if job&.terminated?
+
+      if job.terminated?
         doc[:last_indexed_document_count] = job[:indexed_document_count]
         doc[:last_deleted_document_count] = job[:deleted_document_count]
       end
-      Core::ElasticConnectorActions.update_connector_fields(id, doc)
+
+      Core::ElasticConnectorActions.update_connector_fields(job.connector_id, doc)
     end
 
     private

data/lib/core/elastic_connector_actions.rb

@@ -8,7 +8,6 @@
 #
 require 'active_support/core_ext/hash'
 require 'connectors/connector_status'
-require 'connectors/job_trigger_method'
 require 'connectors/sync_status'
 require 'utility'
 require 'elastic-transport'
@@ -92,17 +91,6 @@ module Core
         )
       end
 
-      def delete_jobs_by_query(query)
-        client.delete_by_query(
-          :index => Utility::Constants::JOB_INDEX,
-          :body => { :query => query }
-        )
-      end
-
-      def delete_indices(indices)
-        client.indices.delete(:index => indices, :ignore_unavailable => true)
-      end
-
       def update_connector_configuration(connector_id, configuration)
         update_connector_fields(connector_id, :configuration => configuration)
       end
@@ -190,15 +178,13 @@ module Core
           status: Connectors::SyncStatus::PENDING,
           created_at: Time.now,
           last_seen: Time.now,
-          trigger_method: connector_settings.sync_now? ? Connectors::JobTriggerMethod::ON_DEMAND : Connectors::JobTriggerMethod::SCHEDULED,
           connector: {
             id: connector_settings.id,
             filtering: convert_connector_filtering_to_job_filtering(connector_settings.filtering),
             index_name: connector_settings.index_name,
             language: connector_settings[:language],
             pipeline: connector_settings[:pipeline],
-            service_type: connector_settings.service_type,
-            configuration: connector_settings.configuration
+            service_type: connector_settings.service_type
           }
         }
 
@@ -234,6 +220,37 @@ module Core
         update_connector_fields(connector_id, body)
       end
 
+      def update_sync(job_id, metadata)
+        body = {
+          :doc => { :last_seen => Time.now }.merge(metadata)
+        }
+        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
+      end
+
+      def complete_sync(connector_id, job_id, metadata, error)
+        sync_status = error ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
+
+        metadata ||= {}
+
+        update_connector_fields(connector_id,
+                                :last_sync_status => sync_status,
+                                :last_sync_error => error,
+                                :error => error,
+                                :last_synced => Time.now,
+                                :last_indexed_document_count => metadata[:indexed_document_count],
+                                :last_deleted_document_count => metadata[:deleted_document_count])
+
+        body = {
+          :doc => {
+            :status => sync_status,
+            :completed_at => Time.now,
+            :last_seen => Time.now,
+            :error => error
+          }.merge(metadata)
+        }
+        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
+      end
+
       def fetch_document_ids(index_name)
         page_size = 1000
         result = []

data/lib/core/scheduler.rb

@@ -78,7 +78,7 @@ module Core
       end
 
       # Sync when sync_now flag is true for the connector
-      if connector_settings.sync_now?
+      if connector_settings[:sync_now] == true
         Utility::Logger.info("#{connector_settings.formatted.capitalize} is manually triggered to sync now.")
         return true
       end
@@ -90,6 +90,13 @@ module Core
         return false
       end
 
+      # We want to sync when sync never actually happened
+      last_synced = connector_settings[:last_synced]
+      if last_synced.nil? || last_synced.empty?
+        Utility::Logger.info("#{connector_settings.formatted.capitalize} has never synced yet, running initial sync.")
+        return true
+      end
+
       current_schedule = scheduling_settings[:interval]
 
       # Don't sync if there is no actual scheduling interval
@@ -112,13 +119,6 @@ module Core
         return false
       end
 
-      # We want to sync when sync never actually happened
-      last_synced = connector_settings[:last_synced]
-      if last_synced.nil? || last_synced.empty?
-        Utility::Logger.info("#{connector_settings.formatted.capitalize} has never synced yet, running initial sync.")
-        return true
-      end
-
       next_trigger_time = cron_parser.next_time(Time.parse(last_synced))
 
       # Sync if next trigger for the connector is in past

data/lib/utility/bulk_queue.rb

@@ -13,7 +13,7 @@ module Utility
     class QueueOverflowError < StandardError; end
 
     # 500 items or 5MB
-    def initialize(operation_count_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_SIZE, size_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES)
+    def initialize(operation_count_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_LENGTH, size_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES)
       @operation_count_threshold = operation_count_threshold.freeze
       @size_threshold = size_threshold.freeze
 

data/lib/utility/error_monitor.rb

@@ -51,7 +51,7 @@ module Utility
     def note_success
       @consecutive_error_count = 0
       @success_count += 1
-      track_window_error(false)
+      increment_window_index
     end
 
     def note_error(error, id: Time.now.to_i)
@@ -60,9 +60,10 @@ module Utility
       Utility::Logger.debug("Message id: #{id} - #{error_message}\n#{stack_trace}")
       @total_error_count += 1
       @consecutive_error_count += 1
+      @window_errors[@window_index] = true
       @error_queue << DocumentError.new(error.class.name, error_message, stack_trace, id)
       @error_queue = @error_queue.drop(1) if @error_queue.size > @error_queue_size
-      track_window_error(true)
+      increment_window_index
       @last_error = error
 
       raise_if_necessary
@@ -91,32 +92,10 @@ module Utility
     end
 
     def num_errors_in_window
-      @window_errors.count(true).to_f
+      @window_errors.count(&:itself).to_f
     end
 
-    def track_window_error(is_error)
-      # We keep the errors array of the size @window_size this way, imagine @window_size = 5
-      # Error array inits as falses:
-      # [ false, false, false, false, false ]
-      # Third document raises an error:
-      # [ false, false, true,  false, false ]
-      #                 ^^^^
-      #                 2 % 5 == 2
-      # Fifth document raises an error:
-      # [ false, false, true,  false, true  ]
-      #                               ^^^^
-      #                               4 % 5 == 4
-      # Sixth document raises an error:
-      # [ true, false, true,   false, true  ]
-      #   ^^^^
-      #   5 % 5 == 0
-      #
-      # Eigth document is successful:
-      # [ true, false, false,  false, true  ]
-      #                ^^^^^
-      #                7 % 5 == 2
-      # And so on.
-      @window_errors[@window_index] = is_error
+    def increment_window_index
       @window_index = (@window_index + 1) % @window_size
     end
 

data/lib/utility/filtering.rb

@@ -17,10 +17,6 @@ module Utility
 
         filter.present? ? filter : {}
       end
-
-      def rule_pre_processing_active?(filter)
-        !filter.dig('advanced_snippet', 'value')&.present?
-      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_utility
 version: !ruby/object:Gem::Version
-  version: 8.6.0.7
+  version: 8.7.0.0.pre.20221117T004939Z
 platform: ruby
 authors:
 - Elastic
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-29 00:00:00.000000000 Z
+date: 2022-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -104,10 +104,8 @@ files:
 - NOTICE.txt
 - lib/connectors/connector_status.rb
 - lib/connectors/crawler/scheduler.rb
-- lib/connectors/job_trigger_method.rb
 - lib/connectors/sync_status.rb
 - lib/connectors_utility.rb
-- lib/core/connector_job.rb
 - lib/core/connector_settings.rb
 - lib/core/elastic_connector_actions.rb
 - lib/core/filtering/validation_status.rb
@@ -132,9 +130,9 @@ homepage: https://github.com/elastic/connectors-ruby
 licenses:
 - Elastic-2.0
 metadata:
-  revision: 86b6c162f41d3a837841d0df2430cbc26b57eb33
-  repository: https://github.com/elastic/connectors-ruby
-post_install_message: 
+  revision: 294214a26b0fe9a4347763b01de681c336e8daae
+  repository: https://github.com/elastic/connectors-ruby.git
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -145,12 +143,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3.1
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Gem containing shared Connector Services libraries
 test_files: []

data/lib/connectors/job_trigger_method.rb

@@ -1,14 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-module Connectors
-  class JobTriggerMethod
-    ON_DEMAND = 'on_demand'
-    SCHEDULED = 'scheduled'
-  end
-end

data/lib/core/connector_job.rb

@@ -1,252 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'active_support/core_ext/hash/indifferent_access'
-require 'connectors/sync_status'
-require 'core/connector_settings'
-require 'core/elastic_connector_actions'
-require 'utility'
-
-module Core
-  class ConnectorJob
-    DEFAULT_PAGE_SIZE = 100
-    STUCK_THRESHOLD = 60
-
-    def self.fetch_by_id(job_id)
-      es_response = ElasticConnectorActions.get_job(job_id)
-      return nil unless es_response[:found]
-
-      new(es_response)
-    end
-
-    def self.pending_jobs(connectors_ids: [], page_size: DEFAULT_PAGE_SIZE)
-      status_term = { status: Connectors::SyncStatus::PENDING_STATUSES }
-
-      query = { bool: { must: [{ terms: status_term }] } }
-
-      return fetch_jobs_by_query(query, page_size) if connectors_ids.empty?
-
-      query[:bool][:must] << { terms: { 'connector.id' => connectors_ids } }
-
-      fetch_jobs_by_query(query, page_size)
-    end
-
-    def self.orphaned_jobs(page_size = DEFAULT_PAGE_SIZE)
-      connector_ids = ConnectorSettings.fetch_all_connectors.map(&:id)
-      query = { bool: { must_not: { terms: { 'connector.id': connector_ids } } } }
-      fetch_jobs_by_query(query, page_size)
-    end
-
-    def self.delete_jobs(jobs)
-      query = { terms: { '_id': jobs.map(&:id) } }
-      ElasticConnectorActions.delete_jobs_by_query(query)
-    end
-
-    def self.stuck_jobs(connector_id = nil, page_size = DEFAULT_PAGE_SIZE)
-      connector_ids = if connector_id
-                        [connector_id]
-                      else
-                        ConnectorSettings.fetch_native_connectors.map(&:id)
-                      end
-      query = {
-        bool: {
-          filter: [
-              { terms: { 'connector.id': connector_ids } },
-              { terms: { status: Connectors::SyncStatus::ACTIVE_STATUSES } },
-              { range: { last_seen: { lte: "now-#{STUCK_THRESHOLD}s" } } }
-          ]
-        }
-      }
-      fetch_jobs_by_query(query, page_size)
-    end
-
-    def self.enqueue(_connector_id)
-      nil
-    end
-
-    def id
-      @elasticsearch_response[:_id]
-    end
-
-    def [](property_name)
-      @elasticsearch_response[:_source][property_name]
-    end
-
-    def error
-      self[:error]
-    end
-
-    def status
-      self[:status]
-    end
-
-    def in_progress?
-      status == Connectors::SyncStatus::IN_PROGRESS
-    end
-
-    def canceling?
-      status == Connectors::SyncStatus::CANCELING
-    end
-
-    def suspended?
-      status == Connectors::SyncStatus::SUSPENDED
-    end
-
-    def canceled?
-      status == Connectors::SyncStatus::CANCELED
-    end
-
-    def pending?
-      Connectors::SyncStatus::PENDING_STATUSES.include?(status)
-    end
-
-    def active?
-      Connectors::SyncStatus::ACTIVE_STATUSES.include?(status)
-    end
-
-    def terminated?
-      Connectors::SyncStatus::TERMINAL_STATUSES.include?(status)
-    end
-
-    def connector_snapshot
-      self[:connector] || {}
-    end
-
-    def connector_id
-      connector_snapshot[:id]
-    end
-
-    def index_name
-      connector_snapshot[:index_name]
-    end
-
-    def language
-      connector_snapshot[:language]
-    end
-
-    def service_type
-      connector_snapshot[:service_type]
-    end
-
-    def configuration
-      connector_snapshot[:configuration]
-    end
-
-    def filtering
-      connector_snapshot[:filtering]
-    end
-
-    def pipeline
-      connector_snapshot[:pipeline] || {}
-    end
-
-    def extract_binary_content?
-      pipeline[:extract_binary_content]
-    end
-
-    def reduce_whitespace?
-      pipeline[:reduce_whitespace]
-    end
-
-    def run_ml_inference?
-      pipeline[:run_ml_inference]
-    end
-
-    def connector
-      @connector ||= ConnectorSettings.fetch_by_id(connector_id)
-    end
-
-    def update_metadata(ingestion_stats = {}, connector_metadata = {})
-      ingestion_stats ||= {}
-      doc = { :last_seen => Time.now }.merge(ingestion_stats)
-      doc[:metadata] = connector_metadata if connector_metadata&.any?
-      ElasticConnectorActions.update_job_fields(id, doc)
-    end
-
-    def done!(ingestion_stats = {}, connector_metadata = {})
-      terminate!(Connectors::SyncStatus::COMPLETED, nil, ingestion_stats, connector_metadata)
-    end
-
-    def error!(message, ingestion_stats = {}, connector_metadata = {})
-      terminate!(Connectors::SyncStatus::ERROR, message, ingestion_stats, connector_metadata)
-    end
-
-    def cancel!(ingestion_stats = {}, connector_metadata = {})
-      terminate!(Connectors::SyncStatus::CANCELED, nil, ingestion_stats, connector_metadata)
-    end
-
-    def with_concurrency_control
-      response = ElasticConnectorActions.get_job(id)
-
-      yield response, response['_seq_no'], response['_primary_term']
-    end
-
-    def make_running!
-      with_concurrency_control do |es_doc, seq_no, primary_term|
-        now = Time.now
-        doc = {
-          status: Connectors::SyncStatus::IN_PROGRESS,
-          started_at: now,
-          last_seen: now,
-          worker_hostname: Socket.gethostname
-        }
-
-        ElasticConnectorActions.update_job_fields(es_doc[:_id], doc, seq_no, primary_term)
-      end
-    end
-
-    def es_source
-      @elasticsearch_response[:_source]
-    end
-
-    private
-
-    def self.fetch_jobs_by_query(query, page_size)
-      results = []
-      offset = 0
-      loop do
-        response = ElasticConnectorActions.search_jobs(query, page_size, offset)
-
-        hits = response.dig('hits', 'hits') || []
-        total = response.dig('hits', 'total', 'value') || 0
-        results += hits.map { |hit| new(hit) }
-        break if results.size >= total
-        offset += hits.size
-      end
-
-      results
-    end
-
-    def initialize(es_response)
-      # TODO: remove the usage of with_indifferent_access. The initialize method should expect a hash argument
-      @elasticsearch_response = es_response.with_indifferent_access
-    end
-
-    def terminate!(status, error = nil, ingestion_stats = {}, connector_metadata = {})
-      ingestion_stats ||= {}
-      ingestion_stats[:total_document_count] = ElasticConnectorActions.document_count(index_name)
-      doc = {
-        :last_seen => Time.now,
-        :completed_at => Time.now,
-        :status => status,
-        :error => error
-      }.merge(ingestion_stats)
-      doc[:canceled_at] = Time.now if status == Connectors::SyncStatus::CANCELED
-      doc[:metadata] = connector_metadata if connector_metadata&.any?
-      ElasticConnectorActions.update_job_fields(id, doc)
-    end
-
-    def seq_no
-      @elasticsearch_response[:_seq_no]
-    end
-
-    def primary_term
-      @elasticsearch_response[:_primary_term]
-    end
-  end
-end