connectors_utility 8.4.0.1 → 8.5.0.0

data/lib/utility/cron.rb

@@ -0,0 +1,81 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'utility/logger'
+
+module Utility
+  # taken from https://regex101.com/r/cU7zG2/1
+  # previous regexp allowed days of the week as [0-6], but it's not correct because the Kibana scheduler
+  # is using [1-7] for days of the week, aligned with the Quartz scheduler: see http://www.quartz-scheduler.org/documentation/2.4.0-SNAPSHOT/tutorials/tutorial-lesson-06.html
+  # But just replacing with [1-7] would also be incorrect, since according to the Cron spec, the days of the week
+  # are 1-6 for Monday-Saturday, and 0 or 7 for Sunday, 7 being a non-standard but still widely used. So, we need to
+  # allow for 0-7.
+  CRON_REGEXP = /^\s*($|#|\w+\s*=|(\?|\*|(?:[0-5]?\d)(?:(?:-|\/|,)(?:[0-5]?\d))?(?:,(?:[0-5]?\d)(?:(?:-|\/|,)(?:[0-5]?\d))?)*)\s+(\?|\*|(?:[0-5]?\d)(?:(?:-|\/|,)(?:[0-5]?\d))?(?:,(?:[0-5]?\d)(?:(?:-|\/|,)(?:[0-5]?\d))?)*)\s+(\?|\*|(?:[01]?\d|2[0-3])(?:(?:-|\/|,)(?:[01]?\d|2[0-3]))?(?:,(?:[01]?\d|2[0-3])(?:(?:-|\/|,)(?:[01]?\d|2[0-3]))?)*)\s+(\?|\*|(?:0?[1-9]|[12]\d|3[01])(?:(?:-|\/|,)(?:0?[1-9]|[12]\d|3[01]))?(?:,(?:0?[1-9]|[12]\d|3[01])(?:(?:-|\/|,)(?:0?[1-9]|[12]\d|3[01]))?)*)\s+(\?|\*|(?:[1-9]|1[012])(?:(?:-|\/|,)(?:[1-9]|1[012]))?(?:L|W)?(?:,(?:[1-9]|1[012])(?:(?:-|\/|,)(?:[1-9]|1[012]))?(?:L|W)?)*|\?|\*|(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)(?:(?:-)(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC))?(?:,(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)(?:(?:-)(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC))?)*)\s+(\?|\*|(?:[0-7])(?:(?:-|\/|,|#)(?:[0-7]))?(?:L)?(?:,(?:[0-7])(?:(?:-|\/|,|#)(?:[0-7]))?(?:L)?)*|\?|\*|(?:MON|TUE|WED|THU|FRI|SAT|SUN)(?:(?:-)(?:MON|TUE|WED|THU|FRI|SAT|SUN))?(?:,(?:MON|TUE|WED|THU|FRI|SAT|SUN)(?:(?:-)(?:MON|TUE|WED|THU|FRI|SAT|SUN))?)*)(|\s)+(\?|\*|(?:|\d{4})(?:(?:-|\/|,)(?:|\d{4}))?(?:,(?:|\d{4})(?:(?:-|\/|,)(?:|\d{4}))?)*))$/
+
+  # see https://github.com/quartz-scheduler/quartz/blob/master/quartz-core/src/main/java/org/quartz/CronExpression.java
+  module Cron
+    def self.check(expr)
+      raise StandardError.new("Unsupported expression #{expr} with #") if expr.include?('#')
+      raise StandardError.new("Unsupported expression #{expr} with L") if expr.include?('L')
+      raise StandardError.new("Unsupported expression #{expr} with W") if expr.include?('W') && !expr.include?('WED')
+
+      expr
+    end
+
+    def self.quartz_to_crontab(expression)
+      @seconds = '*'
+      @minutes = '*'
+      @hours = '*'
+      @day_of_month = '*'
+      @month = '*'
+      @day_of_week = '*'
+      @year = '*'
+
+      # ? is not supported
+      converted_expression = expression.tr('?', '*')
+
+      matched = false
+      converted_expression.match(CRON_REGEXP) { |m|
+        @seconds = m[2]
+        @minutes = m[3]
+        @hours = m[4]
+        @day_of_month = check(m[5])
+        @month = check(m[6])
+        @day_of_week = scheduler_dow_to_crontab(check(m[7])).to_s
+        @year = m[9]
+        matched = true
+      }
+
+      raise StandardError.new("Unknown format #{expression}") unless matched
+
+      # Unix cron has five: minute, hour, day, month, and dayofweek
+      # Quartz adds seconds and year
+      converted_expression = "#{@minutes} #{@hours} #{@day_of_month} #{@month} #{@day_of_week}"
+
+      Utility::Logger.debug("Converted Quartz Cron expression '#{expression}' to Standard Cron Expression '#{converted_expression}'")
+
+      converted_expression
+    end
+
+    # As described above, Quartz uses 1-7 for days of the week, starting with Sunday,
+    # while Unix cron uses 0-6, starting with Monday, and also 7 as an extra non-standard index for Sunday.
+    # (see https://en.wikipedia.org/wiki/Cron for more details)
+    # This means that we need to shift the Quartz day of week that are between 1 and 7 by minus one, but we also allow 0
+    # in case it's not a quartz expression but already the cron standard.
+    # See also the code in connectors-python that does the same thing: https://github.com/elastic/connectors-python/blob/main/connectors/quartz.py
+    def self.scheduler_dow_to_crontab(day)
+      unless /\d/.match?(day)
+        return day
+      end
+      if day.to_i <= 0
+        return day
+      end
+      day.to_i - 1
+    end
+  end
+end

data/lib/utility/environment.rb

@@ -0,0 +1,33 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'logger'
+require 'utility/logger'
+require 'active_support/core_ext/module'
+
+module Utility
+  module Environment
+    def self.set_execution_environment(config, &block)
+      # Set UTC as the timezone
+      ENV['TZ'] = 'UTC'
+      Logger.level = config[:log_level]
+      es_config = config[:elasticsearch]
+      disable_warnings = if es_config.has_key?(:disable_warnings)
+                           es_config[:disable_warnings]
+                         else
+                           true
+                         end
+
+      if disable_warnings
+        Logger.info('Disabling warnings')
+        Kernel.silence_warnings(&block)
+      else
+        Logger.info('Enabling warnings')
+        Kernel.enable_warnings(&block)
+      end
+    end
+  end
+end

data/lib/utility/errors.rb

@@ -0,0 +1,132 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'active_support/core_ext/string'
+
+module Utility
+  class DocumentError
+    attr_accessor :error_class, :error_message, :stack_trace, :error_id
+
+    def initialize(error_class, error_message, stack_trace, error_id)
+      @error_class = error_class
+      @error_message = error_message
+      @error_id = error_id
+
+      # keywords must be < 32kb, UTF-8 chars can be up to 3 bytes, thus 32k/3 ~= 10k
+      # See https://github.com/elastic/workplace-search-team/issues/1723
+      @stack_trace = stack_trace.truncate(10_000)
+    end
+
+    def to_h
+      {
+        'error_class' => error_class,
+        'error_message' => error_message,
+        'stack_trace' => stack_trace,
+        'error_id' => error_id
+      }
+    end
+  end
+
+  class ClientError < StandardError; end
+  class EvictionWithNoProgressError < StandardError; end
+  class EvictionError < StandardError
+    attr_accessor :cursors
+
+    def initialize(message = nil, cursors: nil)
+      super(message)
+      @cursors = cursors
+    end
+  end
+
+  class SuspendedJobError < StandardError
+    attr_accessor :suspend_until, :cursors
+
+    def initialize(message = nil, suspend_until:, cursors: nil)
+      super(message)
+      @suspend_until = suspend_until
+      @cursors = cursors
+    end
+  end
+  class ThrottlingError < SuspendedJobError; end
+  class TransientServerError < SuspendedJobError; end
+  class UnrecoverableServerError < StandardError; end
+  class TransientSubextractorError < StandardError; end
+  class JobDocumentLimitError < StandardError; end
+  class JobClaimingError < StandardError; end
+
+  class MonitoringError < StandardError
+    attr_accessor :tripped_by
+
+    def initialize(message = nil, tripped_by: nil)
+      super("#{message}#{tripped_by.present? ? " Tripped by - #{tripped_by.class}: #{tripped_by.message}" : ''}")
+      @tripped_by = tripped_by
+    end
+  end
+  class MaxSuccessiveErrorsExceededError < MonitoringError; end
+  class MaxErrorsExceededError < MonitoringError; end
+  class MaxErrorsInWindowExceededError < MonitoringError; end
+
+  class JobSyncNotPossibleYetError < StandardError
+    attr_accessor :sync_will_be_possible_at
+
+    def initialize(message = nil, sync_will_be_possible_at: nil)
+      human_readable_errors = []
+
+      human_readable_errors.push(message) unless message.nil?
+      human_readable_errors.push("Content source was created too recently to schedule jobs, next job scheduling is possible at #{sync_will_be_possible_at}.") unless sync_will_be_possible_at.nil?
+
+      super(human_readable_errors.join(' '))
+    end
+  end
+  class PlatinumLicenseRequiredError < StandardError; end
+  class JobInterruptedError < StandardError; end
+  class JobCannotBeUpdatedError < StandardError; end
+  class SecretInvalidError < StandardError; end
+  class InvalidIndexingConfigurationError < StandardError; end
+  class InvalidTokenError < StandardError; end
+  class TokenRefreshFailedError < StandardError; end
+  class ConnectorNotAvailableError < StandardError; end
+
+  # For when we want to explicitly set a #cause but can't
+  class ExplicitlyCausedError < StandardError
+    attr_reader :reason
+
+    def initialize(reason)
+      @reason = reason
+    end
+  end
+
+  class PublishingFailedError < ExplicitlyCausedError; end
+
+  class Error
+    attr_reader :status_code, :code, :message
+
+    def initialize(status_code, code, message)
+      @status_code = status_code
+      @code = code
+      @message = message
+    end
+
+    def to_h
+      {
+        'code' => @code,
+        'message' => @message
+      }
+    end
+  end
+
+  class HealthCheckFailedError < StandardError
+    def initialize(msg = nil)
+      super("Health check failed for 3rd-party service: #{msg}")
+    end
+  end
+
+  INTERNAL_SERVER_ERROR = Utility::Error.new(500, 'INTERNAL_SERVER_ERROR', 'Internal server error')
+  INVALID_API_KEY = Utility::Error.new(401, 'INVALID_API_KEY', 'Invalid API key')
+  UNSUPPORTED_AUTH_SCHEME = Utility::Error.new(401, 'UNSUPPORTED_AUTH_SCHEME', 'Unsupported authorization scheme')
+  INVALID_ACCESS_TOKEN = Utility::Error.new(401, 'INVALID_ACCESS_TOKEN', 'Invalid/expired access token, please refresh the token')
+  TOKEN_REFRESH_ERROR = Utility::Error.new(401, 'TOKEN_REFRESH_ERROR', 'Failed to refresh token, please re-authenticate the application')
+end

data/lib/utility/es_client.rb

@@ -0,0 +1,84 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'logger'
+require 'elasticsearch'
+
+module Utility
+  class EsClient < ::Elasticsearch::Client
+    class IndexingFailedError < StandardError
+      def initialize(message, error = nil)
+        super(message)
+        @cause = error
+      end
+
+      attr_reader :cause
+    end
+
+    def initialize(es_config)
+      super(connection_configs(es_config))
+    end
+
+    def connection_configs(es_config)
+      configs = {}
+      configs[:api_key] = es_config[:api_key] if es_config[:api_key]
+      if es_config[:cloud_id]
+        configs[:cloud_id] = es_config[:cloud_id]
+      elsif es_config[:hosts]
+        configs[:hosts] = es_config[:hosts]
+      else
+        raise 'Either elasticsearch.cloud_id or elasticsearch.hosts should be configured.'
+      end
+      configs[:retry_on_failure] = es_config[:retry_on_failure] || false
+      configs[:request_timeout] = es_config[:request_timeout] || nil
+      configs[:log] = es_config[:log] || false
+      configs[:trace] = es_config[:trace] || false
+
+      # if log or trace is activated, we use the application logger
+      configs[:logger] = if configs[:log] || configs[:trace]
+                           Utility::Logger.logger
+                         else
+                           # silence!
+                           ::Logger.new(IO::NULL)
+                         end
+      configs
+    end
+
+    def bulk(arguments = {})
+      raise_if_necessary(super(arguments))
+    end
+
+    private
+
+    def raise_if_necessary(response)
+      if response['errors']
+        first_error = nil
+
+        response['items'].each do |item|
+          %w[index delete].each do |op|
+            if item.has_key?(op) && item[op].has_key?('error')
+              first_error = item
+
+              break
+            end
+          end
+        end
+
+        if first_error
+          trace_id = Utility::Logger.generate_trace_id
+          Utility::Logger.error("Failed to index documents into Elasticsearch. First error in response is: #{first_error.to_json}")
+          short_message = Utility::Logger.abbreviated_message(first_error.to_json)
+          raise IndexingFailedError.new("Failed to index documents into Elasticsearch with an error '#{short_message}'. Look up the error ID [#{trace_id}] in the application logs to see the full error message.")
+        else
+          raise IndexingFailedError.new('Failed to index documents into Elasticsearch due to unknown error. Try enabling tracing for Elasticsearch and checking the logs.')
+        end
+      end
+      response
+    end
+  end
+end

data/lib/utility/exception_tracking.rb

@@ -0,0 +1,64 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'bson'
+require 'utility/logger'
+
+module Utility
+  class ExceptionTracking
+    class << self
+      def capture_message(message, context = {})
+        Utility::Logger.error("Error: #{message}. Context: #{context.inspect}")
+
+        # When the method is called from a rescue block, our return value may leak outside of its
+        # intended scope, so let's explicitly return nil here to be safe.
+        nil
+      end
+
+      def capture_exception(exception, context = {})
+        Utility::Logger.error(generate_stack_trace(exception))
+        Utility::Logger.error("Context: #{context.inspect}") if context
+      end
+
+      def log_exception(exception, message = nil)
+        Utility::Logger.error(message) if message
+        Utility::Logger.error(generate_stack_trace(exception))
+      end
+
+      def augment_exception(exception)
+        unless exception.respond_to?(:id)
+          exception.instance_eval do
+            def id
+              @error_id ||= BSON::ObjectId.new.to_s
+            end
+          end
+        end
+      end
+
+      def generate_error_message(exception, message, context)
+        context = { :message_id => exception.id }.merge(context || {}) if exception.respond_to?(:id)
+        context_message = context && "Context: #{context.inspect}"
+        ['Exception', message, exception.class.to_s, exception.message, context_message]
+          .compact
+          .map { |part| part.to_s.dup.force_encoding('UTF-8') }
+          .join(': ')
+      end
+
+      def generate_stack_trace(exception)
+        full_message = exception.full_message
+
+        cause = exception
+        while cause.cause != cause && (cause = cause.cause)
+          full_message << "Cause:\n#{cause.full_message}"
+        end
+
+        full_message.dup.force_encoding('UTF-8')
+      end
+    end
+  end
+end

data/lib/utility/extension_mapping_util.rb

@@ -0,0 +1,123 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash'
+
+module Utility
+  class ExtensionMappingUtil
+    @extension_to_mime = {
+      :doc => %w[
+        application/x-tika-msoffice
+        application/msword
+      ].freeze,
+      :docx => %w[
+        application/x-tika-ooxml
+        application/vnd.openxmlformats-officedocument.wordprocessingml.document
+        application/vnd.openxmlformats-officedocument.wordprocessingml.template
+        application/vnd.ms-word.template.macroenabled.12
+        application/vnd.ms-word.document.macroenabled.12
+      ].freeze,
+      :html => %w[
+        text/html
+        application/xhtml+xml
+      ].freeze,
+      :odt => %w[
+        application/x-vnd.oasis.opendocument.graphics-template
+        application/vnd.sun.xml.writer application/x-vnd.oasis.opendocument.text
+        application/x-vnd.oasis.opendocument.text-web
+        application/x-vnd.oasis.opendocument.spreadsheet-template
+        application/vnd.oasis.opendocument.formula-template
+        application/vnd.oasis.opendocument.presentation
+        application/vnd.oasis.opendocument.image-template
+        application/x-vnd.oasis.opendocument.graphics
+        application/vnd.oasis.opendocument.chart-template
+        application/vnd.oasis.opendocument.presentation-template
+        application/x-vnd.oasis.opendocument.image-template
+        application/vnd.oasis.opendocument.formula
+        application/x-vnd.oasis.opendocument.image
+        application/vnd.oasis.opendocument.spreadsheet-template
+        application/x-vnd.oasis.opendocument.chart-template
+        application/x-vnd.oasis.opendocument.formula
+        application/vnd.oasis.opendocument.spreadsheet
+        application/vnd.oasis.opendocument.text-web
+        application/vnd.oasis.opendocument.text-template
+        application/vnd.oasis.opendocument.text
+        application/x-vnd.oasis.opendocument.formula-template
+        application/x-vnd.oasis.opendocument.spreadsheet
+        application/x-vnd.oasis.opendocument.chart
+        application/vnd.oasis.opendocument.text-master
+        application/x-vnd.oasis.opendocument.text-master
+        application/x-vnd.oasis.opendocument.text-template
+        application/vnd.oasis.opendocument.graphics
+        application/vnd.oasis.opendocument.graphics-template
+        application/x-vnd.oasis.opendocument.presentation
+        application/vnd.oasis.opendocument.image
+        application/x-vnd.oasis.opendocument.presentation-template
+        application/vnd.oasis.opendocument.chart
+      ].freeze,
+      :one => %w[
+        application/onenote
+        application/msonenote
+      ].freeze,
+      :pdf => %w[
+        application/pdf
+      ].freeze,
+      :ppt => %w[
+        application/vnd.ms-powerpoint
+      ].freeze,
+      :pptx => %w[
+        application/vnd.openxmlformats-officedocument.presentationml.presentation
+        application/vnd.ms-powerpoint.presentation.macroenabled.12
+        application/vnd.openxmlformats-officedocument.presentationml.template
+        application/vnd.ms-powerpoint.slideshow.macroenabled.12
+        application/vnd.ms-powerpoint.addin.macroenabled.12
+        application/vnd.openxmlformats-officedocument.presentationml.slideshow
+      ].freeze,
+      :rtf => %w[
+        message/richtext
+        text/richtext
+        text/rtf
+        application/rtf
+      ].freeze,
+      :txt => %w[
+        text/plain
+      ].freeze,
+      :xls => %w[
+        application/x-tika-msoffice
+        application/vnd.ms-excel
+        application/vnd.ms-excel.sheet.3
+        application/vnd.ms-excel.sheet.2
+        application/vnd.ms-excel.workspace.3
+        application/vnd.ms-excel.workspace.4
+        application/vnd.ms-excel.sheet.4
+      ].freeze,
+      :xlsx => %w[
+        application/x-tika-ooxml
+        application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+        application/vnd.openxmlformats-officedocument.spreadsheetml.template
+        application/vnd.ms-excel.addin.macroenabled.12
+        application/vnd.ms-excel.template.macroenabled.12
+        application/vnd.ms-excel.sheet.macroenabled.12
+      ].freeze
+    }.with_indifferent_access.freeze
+
+    def self.mime_to_extension
+      @mime_to_extension ||= @extension_to_mime.each_with_object({}) do |(key, values), memo|
+        values.each { |value| memo[value] = key.to_s }
+      end.with_indifferent_access.freeze
+    end
+
+    def self.get_extension(mime_type)
+      mime_to_extension[mime_type.to_s.downcase]
+    end
+
+    def self.get_mime_types(extension)
+      @extension_to_mime[extension.to_s.downcase]
+    end
+  end
+end

data/lib/utility/logger.rb

@@ -0,0 +1,58 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'logger'
+require 'active_support/core_ext/module'
+require 'active_support/core_ext/string/filters'
+require 'ecs_logging/logger'
+
+module Utility
+  class Logger
+    SUPPORTED_LOG_LEVELS = %i[fatal error warn info debug].freeze
+    MAX_SHORT_MESSAGE_LENGTH = 1000.freeze
+
+    class << self
+
+      delegate :formatter, :formatter=, :to => :logger
+
+      def level=(log_level)
+        logger.level = log_level
+      end
+
+      def logger
+        @logger ||= Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
+      end
+
+      SUPPORTED_LOG_LEVELS.each do |level|
+        define_method(level) do |message|
+          if logger.is_a?(EcsLogging::Logger)
+            logger.public_send(level, message, service: { name: 'connectors-ruby' })
+          else
+            logger.public_send(level, message)
+          end
+        end
+      end
+
+      def error_with_backtrace(message: nil, exception: nil, prog_name: nil)
+        logger.error(prog_name) { message } if message
+        logger.error exception.message if exception
+        logger.error exception.backtrace.join("\n") if exception
+      end
+
+      def new_line
+        logger.info("\n")
+      end
+
+      def generate_trace_id
+        SecureRandom.uuid
+      end
+
+      def abbreviated_message(message)
+        message.gsub(/\s+/, ' ').strip.truncate(MAX_SHORT_MESSAGE_LENGTH)
+      end
+    end
+  end
+end

data/lib/utility.rb

@@ -0,0 +1,16 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'utility/constants'
+require 'utility/cron'
+require 'utility/errors'
+require 'utility/es_client'
+require 'utility/environment'
+require 'utility/exception_tracking'
+require 'utility/extension_mapping_util'
+require 'utility/logger'
+require 'utility/elasticsearch/index/mappings'
+require 'utility/elasticsearch/index/text_analysis_settings'