connectors_utility 8.5.0.0 → 8.6.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1aa92692e811978885951049ebd040709d0f286ce4498d93dce54189ff7b8135
-  data.tar.gz: deda3bf664bb03c0b41e75acfa68b4cfa0abce03fd9fd32909c2e0a130686d14
+  metadata.gz: e2ca64052bb1f5b2fc108b3610293b26783d42dc7aa957426bad95415d456527
+  data.tar.gz: c40091f7500e1ef3c39336e1f44a92e6ba9663ad26849af79283543283c3106b
 SHA512:
-  metadata.gz: 648d92295a527ccb6541596e030322cd9d7f6d8858edb90981f635f00f83083667e1e29837649b3310226ba180d0dd2316dc6bcb3bd12d53d4ec7c27f0eeb287
-  data.tar.gz: ac6c2c9b5f1bdf0334f0e7379fe0cddcb7a9cf1d32ed6ad7843da9292cd1e5c025cdd31b10f7c45a7a0d5c23969879ddeee9d7938ecc8a13b4ed577696ba0e08
+  metadata.gz: ca920820800ac34e4e3338c3d0c06799b6e18dcaefd588ef6830bbf5062c453ac30532622dc416b3949e831aee5dd683db68d1227fc2c3ce25bf17ec0ce7da25
+  data.tar.gz: 706728e5dc9e7c0c2dbb6dcdeba46f8efa4efe8ee42028adb47256de710702d928095143f261d92e257ae123797e0e6f9c2da918cf54c76979e70acf1fead187

data/lib/connectors/crawler/scheduler.rb

@@ -21,6 +21,12 @@ module Connectors
         Utility::ExceptionTracking.log_exception(e, 'Could not retrieve Crawler connectors due to unexpected error.')
         []
       end
+
+      private
+
+      def connector_registered?(service_type)
+        service_type == 'elastic-crawler'
+      end
     end
   end
 end

data/lib/core/scheduler.rb

@@ -45,7 +45,7 @@ module Core
         Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
       ensure
         if @poll_interval > 0 && !@is_shutting_down
-          Utility::Logger.info("Sleeping for #{@poll_interval} seconds in #{self.class}.")
+          Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
           sleep(@poll_interval)
         end
       end
@@ -59,29 +59,36 @@ module Core
     private
 
     def sync_triggered?(connector_settings)
+      return false unless connector_registered?(connector_settings.service_type)
+
+      unless connector_settings.valid_index_name?
+        Utility::Logger.warn("The index name of #{connector_settings.formatted} is invalid.")
+        return false
+      end
+
       unless connector_settings.connector_status_allows_sync?
-        Utility::Logger.info("Connector #{connector_settings.id} is in status \"#{connector_settings.connector_status}\" and won't sync yet. Connector needs to be in one of the following statuses: #{Connectors::ConnectorStatus::STATUSES_ALLOWING_SYNC} to run.")
+        Utility::Logger.info("#{connector_settings.formatted.capitalize} is in status \"#{connector_settings.connector_status}\" and won't sync yet. Connector needs to be in one of the following statuses: #{Connectors::ConnectorStatus::STATUSES_ALLOWING_SYNC} to run.")
 
         return false
       end
 
       # Sync when sync_now flag is true for the connector
       if connector_settings[:sync_now] == true
-        Utility::Logger.info("Connector #{connector_settings.id} is manually triggered to sync now.")
+        Utility::Logger.info("#{connector_settings.formatted.capitalize} is manually triggered to sync now.")
         return true
       end
 
       # Don't sync if sync is explicitly disabled
       scheduling_settings = connector_settings.scheduling_settings
       unless scheduling_settings.present? && scheduling_settings[:enabled] == true
-        Utility::Logger.info("Connector #{connector_settings.id} scheduling is disabled.")
+        Utility::Logger.debug("#{connector_settings.formatted.capitalize} scheduling is disabled.")
         return false
       end
 
       # We want to sync when sync never actually happened
       last_synced = connector_settings[:last_synced]
       if last_synced.nil? || last_synced.empty?
-        Utility::Logger.info("Connector #{connector_settings.id} has never synced yet, running initial sync.")
+        Utility::Logger.info("#{connector_settings.formatted.capitalize} has never synced yet, running initial sync.")
         return true
       end
 
@@ -89,7 +96,7 @@ module Core
 
       # Don't sync if there is no actual scheduling interval
       if current_schedule.nil? || current_schedule.empty?
-        Utility::Logger.warn("No sync schedule configured for connector #{connector_settings.id}.")
+        Utility::Logger.warn("No sync schedule configured for #{connector_settings.formatted}.")
         return false
       end
 
@@ -103,7 +110,7 @@ module Core
 
       # Don't sync if the scheduling interval is non-parsable
       unless cron_parser
-        Utility::Logger.error("Unable to parse sync schedule for connector #{connector_settings.id}: expression #{current_schedule} is not a valid Quartz Cron definition.")
+        Utility::Logger.error("Unable to parse sync schedule for #{connector_settings.formatted}: expression #{current_schedule} is not a valid Quartz Cron definition.")
         return false
       end
 
@@ -111,7 +118,7 @@ module Core
 
       # Sync if next trigger for the connector is in past
       if next_trigger_time < Time.now
-        Utility::Logger.info("Connector #{connector_settings.id} sync is triggered by cron schedule #{current_schedule}.")
+        Utility::Logger.info("#{connector_settings.formatted.capitalize} sync is triggered by cron schedule #{current_schedule}.")
         return true
       end
 
@@ -119,6 +126,8 @@ module Core
     end
 
     def heartbeat_triggered?(connector_settings)
+      return false unless connector_registered?(connector_settings.service_type)
+
       last_seen = connector_settings[:last_seen]
       return true if last_seen.nil? || last_seen.empty?
       last_seen = begin
@@ -132,7 +141,20 @@ module Core
     end
 
     def configuration_triggered?(connector_settings)
-      connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
+      if connector_settings.needs_service_type? || connector_registered?(connector_settings.service_type)
+        return connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
+      end
+
+      false
+    end
+
+    def connector_registered?(service_type)
+      if Connectors::REGISTRY.registered?(service_type)
+        true
+      else
+        Utility::Logger.info("The service type (#{service_type}) is not supported.")
+        false
+      end
     end
   end
 end

data/lib/utility/exception_tracking.rb

@@ -21,13 +21,13 @@ module Utility
       end
 
       def capture_exception(exception, context = {})
-        Utility::Logger.error(generate_stack_trace(exception))
+        Utility::Logger.log_stacktrace(generate_stack_trace(exception))
         Utility::Logger.error("Context: #{context.inspect}") if context
       end
 
       def log_exception(exception, message = nil)
         Utility::Logger.error(message) if message
-        Utility::Logger.error(generate_stack_trace(exception))
+        Utility::Logger.log_stacktrace(generate_stack_trace(exception))
       end
 
       def augment_exception(exception)

data/lib/utility/logger.rb

@@ -29,13 +29,21 @@ module Utility
       SUPPORTED_LOG_LEVELS.each do |level|
         define_method(level) do |message|
           if logger.is_a?(EcsLogging::Logger)
-            logger.public_send(level, message, service: { name: 'connectors-ruby' })
+            logger.public_send(level, message, extra_ecs_fields)
           else
             logger.public_send(level, message)
           end
         end
       end
 
+      def log_stacktrace(stacktrace)
+        if logger.is_a?(EcsLogging::Logger)
+          logger.error(nil, extra_ecs_fields.merge(:error => { :stack_trace => stacktrace }))
+        else
+          logger.error(stacktrace)
+        end
+      end
+
       def error_with_backtrace(message: nil, exception: nil, prog_name: nil)
         logger.error(prog_name) { message } if message
         logger.error exception.message if exception
@@ -53,6 +61,24 @@ module Utility
       def abbreviated_message(message)
         message.gsub(/\s+/, ' ').strip.truncate(MAX_SHORT_MESSAGE_LENGTH)
       end
+
+      private
+
+      def extra_ecs_fields
+        {
+          :labels => { :index_date => Time.now.strftime('%Y.%m.%d') },
+          :log => { :logger => logger.progname },
+          :service => {
+            :type => 'connectors-ruby',
+            :version => Settings.version
+          },
+          :process => {
+            :pid => Process.pid,
+            :name => $PROGRAM_NAME,
+            :thread => Thread.current.object_id
+          }
+        }
+      end
     end
   end
 end

metadata

@@ -1,71 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: connectors_utility
 version: !ruby/object:Gem::Version
-  version: 8.5.0.0
+  version: 8.6.0.2
 platform: ruby
 authors:
 - Elastic
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-22 00:00:00.000000000 Z
+date: 2022-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: ecs-logging
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 5.2.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 5.2.6
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: ecs-logging
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.2.6
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.2.6
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
-  name: mime-types
+  name: fugit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: 1.5.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: 1.5.3
 - !ruby/object:Gem::Dependency
-  name: tzinfo-data
+  name: mime-types
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: tzinfo
   requirement: !ruby/object:Gem::Requirement
@@ -81,33 +81,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: tzinfo-data
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.6
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.6
-- !ruby/object:Gem::Dependency
-  name: fugit
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.5.3
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.5.3
+        version: '0'
 description: ''
 email: ent-search-dev@elastic.co
 executables: []
@@ -139,9 +125,9 @@ homepage: https://github.com/elastic/connectors-ruby
 licenses:
 - Elastic-2.0
 metadata:
-  revision: bf9b1793112d26e3f02f75d34874a3e8f0f02f5e
-  repository: git@github.com:elastic/connectors.git
-post_install_message: 
+  revision: d6342a5a66d1f3361ce4a2621d2cf64bad9c8a1c
+  repository: https://github.com/elastic/connectors-ruby.git
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -157,7 +143,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3.1
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Gem containing shared Connector Services libraries
 test_files: []