connectors_utility 8.6.0.6 → 8.6.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ed5c6236d953eba059acb0392bf4a7c29377ac73b5b80d1816da1d445259be60
-  data.tar.gz: d890bf0f8fda93e6c3f34a996a9726488352de8f3a7a7671a6573d626bbcbdb6
+  metadata.gz: d2406276300699ebf07cc2890f994c3a1a9fecb9127793bcad68fb69a4f49111
+  data.tar.gz: 0a35b249367837148c2f7fe2eac30f74de37c5fd5610f30859c703bfbcbb76be
 SHA512:
-  metadata.gz: 02bafd26c7eba58103820a7d086942765fd8f8b6f85c532cc09f819855c2f992417909c1e04a45912163f7bd0ac5eeba90fabcb58b0f34b7c7b2f9b21aaa1988
-  data.tar.gz: 9be7b19622c7c252c28a7ec1bad9d59cfc7e511f13b18b3002a89813fdabb0cdc43573710b23b66f0c239f447c7002fd7f80cb4964cbf689f884044d69031a73
+  metadata.gz: e0cca6acc1faffd3312213d807e77425ed006f44313c3b49faf01c8f3b31817ef7a194249581d6058cc8309773fa1142f5daf8994a2a8e24533f1b3f08f73efd
+  data.tar.gz: '078ebf6f05133d0b3ffe252157e72bcfaee89fb98e65d8ffde1a400c69a95f1ea4edfd7dfd66a626955aa24a0e29217cb6f51ead3e30c1c1d2792d52c7ffbb33'

data/lib/connectors/job_trigger_method.rb

@@ -0,0 +1,14 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Connectors
+  class JobTriggerMethod
+    ON_DEMAND = 'on_demand'
+    SCHEDULED = 'scheduled'
+  end
+end

data/lib/connectors_utility.rb

@@ -9,8 +9,11 @@
 require_relative 'utility'
 
 require_relative 'connectors/connector_status'
+require_relative 'connectors/crawler/scheduler'
+require_relative 'connectors/job_trigger_method'
 require_relative 'connectors/sync_status'
-require_relative 'core/scheduler'
+require_relative 'core/connector_job'
+require_relative 'core/connector_settings'
 require_relative 'core/elastic_connector_actions'
-
-require_relative 'connectors/crawler/scheduler'
+require_relative 'core/filtering/validation_status'
+require_relative 'core/scheduler'

data/lib/core/connector_job.rb

@@ -142,7 +142,19 @@ module Core
     end
 
     def pipeline
-      connector_snapshot[:pipeline]
+      connector_snapshot[:pipeline] || {}
+    end
+
+    def extract_binary_content?
+      pipeline[:extract_binary_content]
+    end
+
+    def reduce_whitespace?
+      pipeline[:reduce_whitespace]
+    end
+
+    def run_ml_inference?
+      pipeline[:run_ml_inference]
     end
 
     def connector

data/lib/core/connector_settings.rb

@@ -8,6 +8,7 @@
 
 require 'active_support/core_ext/hash/indifferent_access'
 require 'connectors/connector_status'
+require 'connectors/sync_status'
 require 'core/elastic_connector_actions'
 require 'utility'
 
@@ -87,6 +88,10 @@ module Core
       self[:scheduling]
     end
 
+    def sync_now?
+      self[:sync_now] == true
+    end
+
     def filtering
       # assume for now, that first object in filtering array or a filter object itself is the only filtering object
       filtering = @elasticsearch_response.dig(:_source, :filtering)
@@ -98,18 +103,6 @@ module Core
       Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
     end
 
-    def extract_binary_content?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
-    end
-
-    def reduce_whitespace?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
-    end
-
-    def run_ml_inference?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
-    end
-
     def formatted
       properties = ["ID: #{id}"]
       properties << "Service type: #{service_type}" if service_type
@@ -139,10 +132,12 @@ module Core
       job_status = job&.status || Connectors::SyncStatus::ERROR
       job_error = job.nil? ? 'Could\'t find the job' : job.error
       job_error ||= 'unknown error' if job_status == Connectors::SyncStatus::ERROR
+      connector_status = (job_status == Connectors::SyncStatus::ERROR ? Connectors::ConnectorStatus::ERROR : Connectors::ConnectorStatus::CONNECTED)
       doc = {
         :last_sync_status => job_status,
         :last_synced => Time.now,
         :last_sync_error => job_error,
+        :status => connector_status,
         :error => job_error
       }
       if job&.terminated?

data/lib/core/elastic_connector_actions.rb

@@ -8,6 +8,7 @@
 #
 require 'active_support/core_ext/hash'
 require 'connectors/connector_status'
+require 'connectors/job_trigger_method'
 require 'connectors/sync_status'
 require 'utility'
 require 'elastic-transport'
@@ -189,13 +190,15 @@ module Core
           status: Connectors::SyncStatus::PENDING,
           created_at: Time.now,
           last_seen: Time.now,
+          trigger_method: connector_settings.sync_now? ? Connectors::JobTriggerMethod::ON_DEMAND : Connectors::JobTriggerMethod::SCHEDULED,
           connector: {
             id: connector_settings.id,
             filtering: convert_connector_filtering_to_job_filtering(connector_settings.filtering),
             index_name: connector_settings.index_name,
             language: connector_settings[:language],
             pipeline: connector_settings[:pipeline],
-            service_type: connector_settings.service_type
+            service_type: connector_settings.service_type,
+            configuration: connector_settings.configuration
           }
         }
 

data/lib/core/scheduler.rb

@@ -78,7 +78,7 @@ module Core
       end
 
       # Sync when sync_now flag is true for the connector
-      if connector_settings[:sync_now] == true
+      if connector_settings.sync_now?
         Utility::Logger.info("#{connector_settings.formatted.capitalize} is manually triggered to sync now.")
         return true
       end

data/lib/utility/error_monitor.rb

@@ -51,7 +51,7 @@ module Utility
     def note_success
       @consecutive_error_count = 0
       @success_count += 1
-      increment_window_index
+      track_window_error(false)
     end
 
     def note_error(error, id: Time.now.to_i)
@@ -60,10 +60,9 @@ module Utility
       Utility::Logger.debug("Message id: #{id} - #{error_message}\n#{stack_trace}")
       @total_error_count += 1
       @consecutive_error_count += 1
-      @window_errors[@window_index] = true
       @error_queue << DocumentError.new(error.class.name, error_message, stack_trace, id)
       @error_queue = @error_queue.drop(1) if @error_queue.size > @error_queue_size
-      increment_window_index
+      track_window_error(true)
       @last_error = error
 
       raise_if_necessary
@@ -92,10 +91,32 @@ module Utility
     end
 
     def num_errors_in_window
-      @window_errors.count(&:itself).to_f
+      @window_errors.count(true).to_f
     end
 
-    def increment_window_index
+    def track_window_error(is_error)
+      # We keep the errors array of the size @window_size this way, imagine @window_size = 5
+      # Error array inits as falses:
+      # [ false, false, false, false, false ]
+      # Third document raises an error:
+      # [ false, false, true,  false, false ]
+      #                 ^^^^
+      #                 2 % 5 == 2
+      # Fifth document raises an error:
+      # [ false, false, true,  false, true  ]
+      #                               ^^^^
+      #                               4 % 5 == 4
+      # Sixth document raises an error:
+      # [ true, false, true,   false, true  ]
+      #   ^^^^
+      #   5 % 5 == 0
+      #
+      # Eigth document is successful:
+      # [ true, false, false,  false, true  ]
+      #                ^^^^^
+      #                7 % 5 == 2
+      # And so on.
+      @window_errors[@window_index] = is_error
       @window_index = (@window_index + 1) % @window_size
     end
 

data/lib/utility/filtering.rb

@@ -17,6 +17,10 @@ module Utility
 
         filter.present? ? filter : {}
       end
+
+      def rule_pre_processing_active?(filter)
+        !filter.dig('advanced_snippet', 'value')&.present?
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_utility
 version: !ruby/object:Gem::Version
-  version: 8.6.0.6
+  version: 8.6.0.7
 platform: ruby
 authors:
 - Elastic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-21 00:00:00.000000000 Z
+date: 2022-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -104,6 +104,7 @@ files:
 - NOTICE.txt
 - lib/connectors/connector_status.rb
 - lib/connectors/crawler/scheduler.rb
+- lib/connectors/job_trigger_method.rb
 - lib/connectors/sync_status.rb
 - lib/connectors_utility.rb
 - lib/core/connector_job.rb
@@ -131,8 +132,8 @@ homepage: https://github.com/elastic/connectors-ruby
 licenses:
 - Elastic-2.0
 metadata:
-  revision: 39cbb85dbae57a2c92e6e0da272d05aa24ca99a9
-  repository: git@github.com:elastic/connectors-ruby.git
+  revision: 86b6c162f41d3a837841d0df2430cbc26b57eb33
+  repository: https://github.com/elastic/connectors-ruby
 post_install_message: 
 rdoc_options: []
 require_paths: