connectors_service 8.5.0.1

data/lib/core/scheduler.rb

@@ -0,0 +1,158 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'time'
+require 'fugit'
+require 'core/connector_settings'
+require 'utility/cron'
+require 'utility/logger'
+require 'utility/exception_tracking'
+
+module Core
+  class Scheduler
+    def initialize(poll_interval, heartbeat_interval)
+      @poll_interval = poll_interval
+      @heartbeat_interval = heartbeat_interval
+      @is_shutting_down = false
+    end
+
+    def connector_settings
+      raise 'Not implemented'
+    end
+
+    def when_triggered
+      loop do
+        connector_settings.each do |cs|
+          if sync_triggered?(cs)
+            yield cs, :sync
+          end
+          if heartbeat_triggered?(cs)
+            yield cs, :heartbeat
+          end
+          if configuration_triggered?(cs)
+            yield cs, :configuration
+          end
+        end
+        if @is_shutting_down
+          break
+        end
+      rescue StandardError => e
+        Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
+      ensure
+        if @poll_interval > 0 && !@is_shutting_down
+          Utility::Logger.info("Sleeping for #{@poll_interval} seconds in #{self.class}.")
+          sleep(@poll_interval)
+        end
+      end
+    end
+
+    def shutdown
+      Utility::Logger.info("Shutting down scheduler #{self.class.name}.")
+      @is_shutting_down = true
+    end
+
+    private
+
+    def sync_triggered?(connector_settings)
+      return false unless connector_registered?(connector_settings.service_type)
+
+      unless connector_settings.valid_index_name?
+        Utility::Logger.info("The index name of #{connector_settings.formatted} is invalid.")
+        return false
+      end
+
+      unless connector_settings.connector_status_allows_sync?
+        Utility::Logger.info("#{connector_settings.formatted.capitalize} is in status \"#{connector_settings.connector_status}\" and won't sync yet. Connector needs to be in one of the following statuses: #{Connectors::ConnectorStatus::STATUSES_ALLOWING_SYNC} to run.")
+
+        return false
+      end
+
+      # Sync when sync_now flag is true for the connector
+      if connector_settings[:sync_now] == true
+        Utility::Logger.info("#{connector_settings.formatted.capitalize} is manually triggered to sync now.")
+        return true
+      end
+
+      # Don't sync if sync is explicitly disabled
+      scheduling_settings = connector_settings.scheduling_settings
+      unless scheduling_settings.present? && scheduling_settings[:enabled] == true
+        Utility::Logger.info("#{connector_settings.formatted.capitalize} scheduling is disabled.")
+        return false
+      end
+
+      # We want to sync when sync never actually happened
+      last_synced = connector_settings[:last_synced]
+      if last_synced.nil? || last_synced.empty?
+        Utility::Logger.info("#{connector_settings.formatted.capitalize} has never synced yet, running initial sync.")
+        return true
+      end
+
+      current_schedule = scheduling_settings[:interval]
+
+      # Don't sync if there is no actual scheduling interval
+      if current_schedule.nil? || current_schedule.empty?
+        Utility::Logger.warn("No sync schedule configured for #{connector_settings.formatted}.")
+        return false
+      end
+
+      current_schedule = begin
+        Utility::Cron.quartz_to_crontab(current_schedule)
+      rescue StandardError => e
+        Utility::ExceptionTracking.log_exception(e, "Unable to convert quartz (#{current_schedule}) to crontab.")
+        return false
+      end
+      cron_parser = Fugit::Cron.parse(current_schedule)
+
+      # Don't sync if the scheduling interval is non-parsable
+      unless cron_parser
+        Utility::Logger.error("Unable to parse sync schedule for #{connector_settings.formatted}: expression #{current_schedule} is not a valid Quartz Cron definition.")
+        return false
+      end
+
+      next_trigger_time = cron_parser.next_time(Time.parse(last_synced))
+
+      # Sync if next trigger for the connector is in past
+      if next_trigger_time < Time.now
+        Utility::Logger.info("#{connector_settings.formatted.capitalize} sync is triggered by cron schedule #{current_schedule}.")
+        return true
+      end
+
+      false
+    end
+
+    def heartbeat_triggered?(connector_settings)
+      return false unless connector_registered?(connector_settings.service_type)
+
+      last_seen = connector_settings[:last_seen]
+      return true if last_seen.nil? || last_seen.empty?
+      last_seen = begin
+        Time.parse(last_seen)
+      rescue StandardError
+        Utility::Logger.warn("Unable to parse last_seen #{last_seen}")
+        nil
+      end
+      return true unless last_seen
+      last_seen + @heartbeat_interval < Time.now
+    end
+
+    def configuration_triggered?(connector_settings)
+      return false unless connector_registered?(connector_settings.service_type)
+
+      connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
+    end
+
+    def connector_registered?(service_type)
+      if Connectors::REGISTRY.registered?(service_type)
+        true
+      else
+        Utility::Logger.info("The service type (#{service_type}) is not supported.")
+        false
+      end
+    end
+  end
+end

data/lib/core/single_scheduler.rb

@@ -0,0 +1,29 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/scheduler'
+require 'core/connector_settings'
+require 'utility/logger'
+require 'utility/exception_tracking'
+
+module Core
+  class SingleScheduler < Core::Scheduler
+    def initialize(connector_id, poll_interval, heartbeat_interval)
+      super(poll_interval, heartbeat_interval)
+      @connector_id = connector_id
+    end
+
+    def connector_settings
+      connector_settings = Core::ConnectorSettings.fetch_by_id(@connector_id)
+      [connector_settings]
+    rescue StandardError => e
+      Utility::ExceptionTracking.log_exception(e, "Could not retrieve the connector by id #{@connector_id} due to unexpected error.")
+      []
+    end
+  end
+end

data/lib/core/sync_job_runner.rb

@@ -0,0 +1,111 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/connector_status'
+require 'connectors/registry'
+require 'core/output_sink'
+require 'utility'
+
+module Core
+  class IncompatibleConfigurableFieldsError < StandardError
+    def initialize(service_type, expected_fields, actual_fields)
+      super("Connector of service_type '#{service_type}' expected configurable fields: #{expected_fields}, actual stored fields: #{actual_fields}")
+    end
+  end
+
+  class SyncJobRunner
+    def initialize(connector_settings)
+      @connector_settings = connector_settings
+      @sink = Core::OutputSink::EsSink.new(connector_settings.index_name, @connector_settings.request_pipeline)
+      @connector_class = Connectors::REGISTRY.connector_class(connector_settings.service_type)
+      @connector_instance = Connectors::REGISTRY.connector(connector_settings.service_type, connector_settings.configuration)
+      @status = {
+        :indexed_document_count => 0,
+        :deleted_document_count => 0,
+        :error => nil
+      }
+    end
+
+    def execute
+      validate_configuration!
+      do_sync!
+    end
+
+    private
+
+    def do_sync!
+      Utility::Logger.info("Starting sync for connector #{@connector_settings.id}.")
+
+      job_id = ElasticConnectorActions.claim_job(@connector_settings.id)
+
+      unless job_id.present?
+        Utility::Logger.error("Failed to claim the job for #{@connector_settings.id}. Please check the logs for the cause of this error.")
+        return
+      end
+
+      begin
+        Utility::Logger.debug("Successfully claimed job for connector #{@connector_settings.id}.")
+
+        @connector_instance.do_health_check!
+
+        incoming_ids = []
+        existing_ids = ElasticConnectorActions.fetch_document_ids(@connector_settings.index_name)
+
+        Utility::Logger.debug("#{existing_ids.size} documents are present in index #{@connector_settings.index_name}.")
+
+        @connector_instance.yield_documents do |document|
+          document = add_ingest_metadata(document)
+          @sink.ingest(document)
+          incoming_ids << document[:id]
+          @status[:indexed_document_count] += 1
+        end
+
+        ids_to_delete = existing_ids - incoming_ids.uniq
+
+        Utility::Logger.info("Deleting #{ids_to_delete.size} documents from index #{@connector_settings.index_name}.")
+
+        ids_to_delete.each do |id|
+          @sink.delete(id)
+          @status[:deleted_document_count] += 1
+        end
+
+        @sink.flush
+      rescue StandardError => e
+        @status[:error] = e.message
+        Utility::ExceptionTracking.log_exception(e)
+        ElasticConnectorActions.update_connector_status(@connector_settings.id, Connectors::ConnectorStatus::ERROR, Utility::Logger.abbreviated_message(e.message))
+      ensure
+        Utility::Logger.info("Upserted #{@status[:indexed_document_count]} documents into #{@connector_settings.index_name}.")
+        Utility::Logger.info("Deleted #{@status[:deleted_document_count]} documents into #{@connector_settings.index_name}.")
+
+        ElasticConnectorActions.complete_sync(@connector_settings.id, job_id, @status.dup)
+
+        if @status[:error]
+          Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error #{@status[:error]}.")
+        else
+          Utility::Logger.info("Successfully synced for connector #{@connector_settings.id}.")
+        end
+      end
+    end
+
+    def add_ingest_metadata(document)
+      document.tap do |it|
+        it['_extract_binary_content'] = @connector_settings.extract_binary_content? if @connector_settings.extract_binary_content?
+        it['_reduce_whitespace'] = @connector_settings.reduce_whitespace? if @connector_settings.reduce_whitespace?
+        it['_run_ml_inference'] = @connector_settings.run_ml_inference? if @connector_settings.run_ml_inference?
+      end
+    end
+
+    def validate_configuration!
+      expected_fields = @connector_class.configurable_fields.keys.map(&:to_s).sort
+      actual_fields = @connector_settings.configuration.keys.map(&:to_s).sort
+
+      raise IncompatibleConfigurableFieldsError.new(@connector_class.service_type, expected_fields, actual_fields) if expected_fields != actual_fields
+    end
+  end
+end

data/lib/core.rb

@@ -0,0 +1,16 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/configuration'
+require 'core/connector_settings'
+require 'core/elastic_connector_actions'
+require 'core/heartbeat'
+require 'core/scheduler'
+require 'core/single_scheduler'
+require 'core/native_scheduler'
+require 'core/sync_job_runner'

data/lib/list_connectors.rb

@@ -0,0 +1,22 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/registry'
+require 'utility'
+
+class ListConnectors
+  def self.run!
+    Utility::Environment.set_execution_environment(App::Config) do
+      Utility::Logger.info('Registered connectors:')
+      Connectors::REGISTRY.registered_connectors.each do |connector|
+        Utility::Logger.info("- #{Connectors::REGISTRY.connector_class(connector).display_name}")
+      end
+      Utility::Logger.info('Bye')
+    end
+  end
+end

data/lib/stubs/app_config.rb

@@ -0,0 +1,35 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+class AppConfig
+  class << self
+    def connectors
+      {
+        'transient_server_error_retry_delay_minutes' => 5
+      }
+    end
+
+    def content_source_sync_max_errors
+      1000
+    end
+
+    def content_source_sync_max_consecutive_errors
+      10
+    end
+
+    def content_source_sync_max_error_ratio
+      0.15
+    end
+
+    def content_source_sync_error_ratio_window_size
+      100
+    end
+
+    def content_source_sync_thumbnails_enabled?
+      true
+    end
+  end
+end

data/lib/stubs/connectors/stats.rb

@@ -0,0 +1,35 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/inflector'
+
+module Connectors
+  module Stats
+    def self.measure(_key, _value = nil, &block)
+      block.call
+    end
+
+    def self.increment(key, value = 1)
+      # no op
+    end
+
+    def self.prefix_key(key)
+      "connectors.#{key}"
+    end
+
+    def self.class_key(klass, deconstantize = true)
+      name = klass.name
+      # Changes Connectors::GoogleDrive::Adapter to Connectors::GoogleDrive
+      name = ActiveSupport::Inflector.deconstantize(name) if deconstantize
+      # Changes Connectors::GoogleDrive to GoogleDrive
+      name = ActiveSupport::Inflector.demodulize(name)
+      # Changes GoogleDrive to google_drive
+      ActiveSupport::Inflector.underscore(name)
+    end
+  end
+end

data/lib/stubs/service_type.rb

@@ -0,0 +1,13 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+class ServiceType
+  def classify
+    'classify'
+  end
+end

data/lib/utility/constants.rb

@@ -0,0 +1,20 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Utility
+  class Constants
+    THUMBNAIL_FIELDS = %w[_thumbnail_80x100 _thumbnail_310x430].freeze
+    SUBEXTRACTOR_RESERVED_FIELDS = %w[_subextracted_as_of _subextracted_version].freeze
+    ALLOW_FIELD = '_allow_permissions'
+    DENY_FIELD = '_deny_permissions'
+    CONNECTORS_INDEX = '.elastic-connectors'
+    JOB_INDEX = '.elastic-connectors-sync-jobs'
+    CONTENT_INDEX_PREFIX = 'search-'
+    CRAWLER_SERVICE_TYPE = 'elastic-crawler'
+  end
+end

data/lib/utility/cron.rb

@@ -0,0 +1,81 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'utility/logger'
+
+module Utility
+  # taken from https://regex101.com/r/cU7zG2/1
+  # previous regexp allowed days of the week as [0-6], but it's not correct because the Kibana scheduler
+  # is using [1-7] for days of the week, aligned with the Quartz scheduler: see http://www.quartz-scheduler.org/documentation/2.4.0-SNAPSHOT/tutorials/tutorial-lesson-06.html
+  # But just replacing with [1-7] would also be incorrect, since according to the Cron spec, the days of the week
+  # are 1-6 for Monday-Saturday, and 0 or 7 for Sunday, 7 being a non-standard but still widely used. So, we need to
+  # allow for 0-7.
+  CRON_REGEXP = /^\s*($|#|\w+\s*=|(\?|\*|(?:[0-5]?\d)(?:(?:-|\/|,)(?:[0-5]?\d))?(?:,(?:[0-5]?\d)(?:(?:-|\/|,)(?:[0-5]?\d))?)*)\s+(\?|\*|(?:[0-5]?\d)(?:(?:-|\/|,)(?:[0-5]?\d))?(?:,(?:[0-5]?\d)(?:(?:-|\/|,)(?:[0-5]?\d))?)*)\s+(\?|\*|(?:[01]?\d|2[0-3])(?:(?:-|\/|,)(?:[01]?\d|2[0-3]))?(?:,(?:[01]?\d|2[0-3])(?:(?:-|\/|,)(?:[01]?\d|2[0-3]))?)*)\s+(\?|\*|(?:0?[1-9]|[12]\d|3[01])(?:(?:-|\/|,)(?:0?[1-9]|[12]\d|3[01]))?(?:,(?:0?[1-9]|[12]\d|3[01])(?:(?:-|\/|,)(?:0?[1-9]|[12]\d|3[01]))?)*)\s+(\?|\*|(?:[1-9]|1[012])(?:(?:-|\/|,)(?:[1-9]|1[012]))?(?:L|W)?(?:,(?:[1-9]|1[012])(?:(?:-|\/|,)(?:[1-9]|1[012]))?(?:L|W)?)*|\?|\*|(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)(?:(?:-)(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC))?(?:,(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)(?:(?:-)(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC))?)*)\s+(\?|\*|(?:[0-7])(?:(?:-|\/|,|#)(?:[0-7]))?(?:L)?(?:,(?:[0-7])(?:(?:-|\/|,|#)(?:[0-7]))?(?:L)?)*|\?|\*|(?:MON|TUE|WED|THU|FRI|SAT|SUN)(?:(?:-)(?:MON|TUE|WED|THU|FRI|SAT|SUN))?(?:,(?:MON|TUE|WED|THU|FRI|SAT|SUN)(?:(?:-)(?:MON|TUE|WED|THU|FRI|SAT|SUN))?)*)(|\s)+(\?|\*|(?:|\d{4})(?:(?:-|\/|,)(?:|\d{4}))?(?:,(?:|\d{4})(?:(?:-|\/|,)(?:|\d{4}))?)*))$/
+
+  # see https://github.com/quartz-scheduler/quartz/blob/master/quartz-core/src/main/java/org/quartz/CronExpression.java
+  module Cron
+    def self.check(expr)
+      raise StandardError.new("Unsupported expression #{expr} with #") if expr.include?('#')
+      raise StandardError.new("Unsupported expression #{expr} with L") if expr.include?('L')
+      raise StandardError.new("Unsupported expression #{expr} with W") if expr.include?('W') && !expr.include?('WED')
+
+      expr
+    end
+
+    def self.quartz_to_crontab(expression)
+      @seconds = '*'
+      @minutes = '*'
+      @hours = '*'
+      @day_of_month = '*'
+      @month = '*'
+      @day_of_week = '*'
+      @year = '*'
+
+      # ? is not supported
+      converted_expression = expression.tr('?', '*')
+
+      matched = false
+      converted_expression.match(CRON_REGEXP) { |m|
+        @seconds = m[2]
+        @minutes = m[3]
+        @hours = m[4]
+        @day_of_month = check(m[5])
+        @month = check(m[6])
+        @day_of_week = scheduler_dow_to_crontab(check(m[7])).to_s
+        @year = m[9]
+        matched = true
+      }
+
+      raise StandardError.new("Unknown format #{expression}") unless matched
+
+      # Unix cron has five: minute, hour, day, month, and dayofweek
+      # Quartz adds seconds and year
+      converted_expression = "#{@minutes} #{@hours} #{@day_of_month} #{@month} #{@day_of_week}"
+
+      Utility::Logger.debug("Converted Quartz Cron expression '#{expression}' to Standard Cron Expression '#{converted_expression}'")
+
+      converted_expression
+    end
+
+    # As described above, Quartz uses 1-7 for days of the week, starting with Sunday,
+    # while Unix cron uses 0-6, starting with Monday, and also 7 as an extra non-standard index for Sunday.
+    # (see https://en.wikipedia.org/wiki/Cron for more details)
+    # This means that we need to shift the Quartz day of week that are between 1 and 7 by minus one, but we also allow 0
+    # in case it's not a quartz expression but already the cron standard.
+    # See also the code in connectors-python that does the same thing: https://github.com/elastic/connectors-python/blob/main/connectors/quartz.py
+    def self.scheduler_dow_to_crontab(day)
+      unless /\d/.match?(day)
+        return day
+      end
+      if day.to_i <= 0
+        return day
+      end
+      day.to_i - 1
+    end
+  end
+end

data/lib/utility/elasticsearch/index/language_data.yml

@@ -0,0 +1,111 @@
+---
+da:
+  name: Danish
+  stemmer: danish
+  stop_words: _danish_
+de:
+  name: German
+  stemmer: light_german
+  stop_words: _german_
+en:
+  name: English
+  stemmer: light_english
+  stop_words: _english_
+es:
+  name: Spanish
+  stemmer: light_spanish
+  stop_words: _spanish_
+fr:
+  name: French
+  stemmer: light_french
+  stop_words: _french_
+  custom_filter_definitions:
+    fr-elision:
+      type: elision
+      articles:
+      - l
+      - m
+      - t
+      - qu
+      - n
+      - s
+      - j
+      - d
+      - c
+      - jusqu
+      - quoiqu
+      - lorsqu
+      - puisqu
+      articles_case: true
+  prepended_filters:
+  - fr-elision
+it:
+  name: Italian
+  stemmer: light_italian
+  stop_words: _italian_
+  custom_filter_definitions:
+    it-elision:
+      type: elision
+      articles:
+      - c
+      - l
+      - all
+      - dall
+      - dell
+      - nell
+      - sull
+      - coll
+      - pell
+      - gl
+      - agl
+      - dagl
+      - degl
+      - negl
+      - sugl
+      - un
+      - m
+      - t
+      - s
+      - v
+      - d
+      articles_case: true
+  prepended_filters:
+  - it-elision
+ja:
+  name: Japanese
+  stemmer: light_english
+  stop_words: _english_
+  postpended_filters:
+  - cjk_bigram
+ko:
+  name: Korean
+  stemmer: light_english
+  stop_words: _english_
+  postpended_filters:
+  - cjk_bigram
+nl:
+  name: Dutch
+  stemmer: dutch
+  stop_words: _dutch_
+pt:
+  name: Portuguese
+  stemmer: light_portuguese
+  stop_words: _portuguese_
+pt-br:
+  name: Portuguese (Brazil)
+  stemmer: brazilian
+  stop_words: _brazilian_
+ru:
+  name: Russian
+  stemmer: russian
+  stop_words: _russian_
+th:
+  name: Thai
+  stemmer: light_english
+  stop_words: _thai_
+zh:
+  name: Chinese
+  stemmer: light_english
+  stop_words: _english_
+  postpended_filters:
+  - cjk_bigram