connectors_sdk 8.3.0.0.pre.20220414T060419Z

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a44a80ba5fe5032b6254fb8a2db25516542df5818044d627b6ba31f652a7812f
+  data.tar.gz: 52d1245cc24ab6e1a42c00483ec4a8ece48ede4a6465c9b46b3bf5e18c979678
+SHA512:
+  metadata.gz: 7a4021e1d7b83ec7334913f1ef5e684e02296097f223e96610cb31eb5502c36f9b45b03eeaf80e876aac5bcc531f85d10445917459b86cedc30d6609b33f5351
+  data.tar.gz: ca04ab72f48ed6e76b3a0e0cc8c260f81779ea906325a782eb61456c9c5be231043d7a73ca9625df63459460de587c65aa352fb2e1150fc8c92a253536fcd634

data/LICENSE

@@ -0,0 +1,93 @@
+Elastic License 2.0
+
+URL: https://www.elastic.co/licensing/elastic-license
+
+## Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+## Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide,
+non-sublicensable, non-transferable license to use, copy, distribute, make
+available, and prepare derivative works of the software, in each case subject to
+the limitations and conditions below.
+
+## Limitations
+
+You may not provide the software to third parties as a hosted or managed
+service, where the service provides users with access to any substantial set of
+the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality
+in the software, and you may not remove or obscure any functionality in the
+software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices
+of the licensor in the software. Any use of the licensor’s trademarks is subject
+to applicable law.
+
+## Patents
+
+The licensor grants you a license, under any patent claims the licensor can
+license, or becomes able to license, to make, have made, use, sell, offer for
+sale, import and have imported the software, in each case subject to the
+limitations and conditions in this license. This license does not cover any
+patent claims that you cause to be infringed by modifications or additions to
+the software. If you or your company make any written claim that the software
+infringes or contributes to infringement of any patent, your patent license for
+the software granted under these terms ends immediately. If your company makes
+such a claim, your patent license ends immediately for work on behalf of your
+company.
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you
+also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the
+software prominent notices stating that you have modified the software.
+
+## No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in
+these terms.
+
+## Termination
+
+If you use the software in violation of these terms, such use is not licensed,
+and your licenses will automatically terminate. If the licensor provides you
+with a notice of your violation, and you cease all violation of this license no
+later than 30 days after you receive that notice, your licenses will be
+reinstated retroactively. However, if you violate these terms after such
+reinstatement, any additional violation of these terms will cause your licenses
+to terminate automatically and permanently.
+
+## No Liability
+
+*As far as the law allows, the software comes as is, without any warranty or
+condition, and the licensor will not be liable to you for any damages arising
+out of these terms or the use or nature of the software, under any kind of
+legal claim.*
+
+## Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the
+software the licensor makes available under these terms, including any portion
+of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of
+organization that you work for, plus all organizations that have control over,
+are under the control of, or are under common control with that
+organization. **control** means ownership of substantially all the assets of an
+entity, or the power to direct its management and policies by vote, contract, or
+otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under
+these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.

data/NOTICE.txt

@@ -0,0 +1,2 @@
+connectors
+Copyright 2022 Elasticsearch B.V.

data/lib/connectors_sdk/base/adapter.rb

@@ -0,0 +1,118 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'active_support/core_ext/array/wrap'
+require 'active_support/core_ext/numeric/time'
+require 'active_support/core_ext/object/deep_dup'
+require 'connectors_shared'
+require 'connectors_shared/extension_mapping_util'
+require 'date'
+require 'active_support/all'
+require 'mime-types'
+
+module ConnectorsSdk
+  module Base
+    class Adapter
+      def self.fields_to_preserve
+        @fields_to_preserve ||= ['body']
+          .concat(ConnectorsShared::Constants::THUMBNAIL_FIELDS)
+          .concat(ConnectorsShared::Constants::SUBEXTRACTOR_RESERVED_FIELDS)
+          .map(&:freeze)
+          .freeze
+      end
+
+      def self.generate_id_helpers(method_prefix, id_prefix)
+        define_singleton_method("#{method_prefix}_id_to_fp_id") do |id|
+          "#{id_prefix}_#{id}"
+        end
+
+        define_singleton_method("fp_id_is_#{method_prefix}_id?") do |fp_id|
+          regex_match = /#{id_prefix}_(.+)$/.match(fp_id)
+          regex_match.present? && regex_match.size == 2
+        end
+
+        define_singleton_method("fp_id_to_#{method_prefix}_id") do |fp_id|
+          regex_match = /#{id_prefix}_(.+)$/.match(fp_id)
+
+          raise ArgumentError, "Invalid id #{fp_id} for source with method prefix #{method_prefix}." if regex_match.nil? || regex_match.length != 2
+          regex_match[1]
+        end
+      end
+
+      def self.mime_type_for_file(file_name)
+        ruby_detected_type = MIME::Types.type_for(file_name)
+        return ruby_detected_type.first.simplified if ruby_detected_type.present?
+        extension = extension_for_file(file_name)
+        ConnectorsShared::ExtensionMappingUtil.get_mime_types(extension)&.first
+      end
+
+      def self.extension_for_file(file_name)
+        File.extname(file_name.downcase).delete_prefix!('.')
+      end
+
+      def self.strip_file_extension(file_name)
+        File.basename(file_name, File.extname(file_name))
+      end
+
+      def self.normalize_enum(enum)
+        enum&.to_s&.downcase
+      end
+
+      def self.normalize_date(date)
+        return nil if date.blank?
+
+        case date
+        when Date, Time, DateTime, ActiveSupport::TimeWithZone
+          date.to_datetime.rfc3339
+        else
+          begin
+            Time.zone.parse(date).to_datetime.rfc3339
+          rescue ArgumentError, TypeError => e
+            ConnectorsShared::ExceptionTracking.capture_exception(e)
+            nil
+          end
+        end
+      end
+
+      def self.normalize_path(path)
+        return nil if path.blank?
+        return path if path.start_with?('/')
+        "/#{path}"
+      end
+
+      def self.url_to_path(url)
+        return nil if url.blank?
+        uri = URI(url)
+        return nil if uri.scheme.blank?
+        normalize_path(uri.path)
+      rescue URI::InvalidURIError, ArgumentError
+        nil
+      end
+
+      def self.swiftype_document_from_configured_object_base(object_type:, object:, fields:)
+        object_as_json = object.as_json
+
+        adapted_object = {
+          :type => normalize_enum(object_type)
+        }
+
+        fields.each do |field_data|
+          remote_field_name = field_data.fetch(:remote)
+
+          value = object_as_json[remote_field_name]
+          value = object_as_json.dig(*remote_field_name.split('.')) if value.blank?
+          next if value.nil?
+
+          adapted_object[field_data.fetch(:target)] = value
+        end
+
+        adapted_object.symbolize_keys
+      end
+
+      delegate :normalize_enum, :normalize_date, :normalize_path, :to => :class
+    end
+  end
+end

data/lib/connectors_sdk/base/config.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+module ConnectorsSdk
+  module Base
+    class Config
+      attr_reader :cursors
+
+      def initialize(cursors:)
+        @cursors = cursors || {}
+      end
+
+      def to_h
+        {
+          :cursors => cursors
+        }
+      end
+
+      def overwrite_cursors!(new_cursors)
+        @cursors = new_cursors
+      end
+    end
+  end
+end

data/lib/connectors_sdk/base/custom_client.rb

@@ -0,0 +1,112 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'faraday'
+require 'httpclient'
+require 'active_support/core_ext/array/wrap'
+require 'active_support/core_ext/numeric/time'
+require 'active_support/core_ext/object/deep_dup'
+require 'connectors_shared'
+require 'date'
+require 'active_support/all'
+
+module ConnectorsSdk
+  module Base
+    class CustomClient
+      attr_reader :base_url, :middleware, :ensure_fresh_auth
+
+      MAX_RETRIES = 5
+
+      def initialize(base_url: nil, ensure_fresh_auth: nil)
+        @base_url = base_url
+        @ensure_fresh_auth = ensure_fresh_auth
+        middleware!
+      end
+
+      def middleware!
+        @middleware = Array.wrap(additional_middleware)
+        @middleware += Array.wrap(default_middleware)
+        @middleware.compact!
+      end
+
+      def additional_middleware
+        [] # define as needed in subclass
+      end
+
+      def default_middleware
+        [[Faraday::Request::Retry, retry_config]]
+      end
+
+      def retry_config
+        {
+          :retry_statuses => [429],
+          :backoff_factor => 2,
+          :max => MAX_RETRIES,
+          :interval => 0.05
+        }
+      end
+
+      [
+        :delete,
+        :get,
+        :head,
+        :options,
+        :patch,
+        :post,
+        :put,
+      ].each do |http_verb|
+        define_method http_verb do |*args, &block|
+          ensure_fresh_auth.call(self) if ensure_fresh_auth.present?
+          http_client.public_send(http_verb, *args, &block)
+        end
+      end
+
+      def http_client!
+        @http_client = nil
+        http_client
+      end
+
+      def http_client
+        @http_client ||= Faraday.new(base_url) do |faraday|
+          middleware.each do |middleware_config|
+            faraday.use(*middleware_config)
+          end
+
+          faraday.adapter(:httpclient)
+        end
+      end
+
+      private
+
+      # https://github.com/lostisland/faraday/blob/b09c6db31591dd1a58fffcc0979b0c7d96b5388b/lib/faraday/connection.rb#L171
+      METHODS_WITH_BODY = [:post, :put, :patch].freeze
+
+      def send_body?(method)
+        METHODS_WITH_BODY.include?(method.to_sym)
+      end
+
+      def request_with_throttling(method, url, options = {})
+        response =
+          if send_body?(method)
+            public_send(method, url, options[:body], options[:headers])
+          else
+            public_send(method, url, options[:params], options[:headers])
+          end
+
+        if response.status == 429
+          retry_after = response.headers['Retry-After']
+          multiplier = options.fetch(:retry_mulitplier, 1)
+          retry_after_secs = (retry_after.is_a?(Array) ? retry_after.first.to_i : retry_after.to_i) * multiplier
+          retry_after_secs = 60 if retry_after_secs <= 0
+          ConnectorsShared::Logger.warn("Exceeded #{self.class} request limits. Going to sleep for #{retry_after_secs} seconds")
+          raise ConnectorsShared::ThrottlingError.new(:suspend_until => DateTime.now + retry_after_secs.seconds, :cursors => options[:cursors])
+        else
+          response
+        end
+      end
+    end
+  end
+end

data/lib/connectors_sdk/base/extractor.rb

@@ -0,0 +1,256 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'faraday'
+require 'httpclient'
+require 'active_support/core_ext/array/wrap'
+require 'active_support/core_ext/numeric/time'
+require 'active_support/core_ext/object/deep_dup'
+require 'connectors_shared'
+require 'date'
+require 'active_support/all'
+require 'stubs/connectors/stats' unless defined?(Rails)
+
+module ConnectorsSdk
+  module Base
+    class Extractor
+      MAX_CONNECTION_ATTEMPTS = 3
+      DEFAULT_CURSOR_KEY = 'all'.freeze
+
+      TRANSIENT_SERVER_ERROR_CLASSES = Set.new(
+        [
+          Faraday::ConnectionFailed,
+          Faraday::SSLError,
+          Faraday::TimeoutError,
+          HTTPClient::ConnectTimeoutError,
+          Net::OpenTimeout
+        ]
+      )
+
+      attr_reader :content_source_id, :config, :features, :original_cursors, :service_type, :completed
+      attr_accessor :monitor, :client_proc
+
+      def initialize(content_source_id:,
+                     service_type:,
+                     config:,
+                     features:,
+                     client_proc:,
+                     authorization_data_proc:,
+                     monitor: ConnectorsShared::Monitor.new(:connector => self))
+        @content_source_id = content_source_id
+        @service_type = service_type
+        @config = config
+        @features = features
+        @client_proc = client_proc
+        @authorization_data_proc = authorization_data_proc
+        @original_cursors = config.cursors.deep_dup
+        @monitor = monitor
+        @completed = false
+      end
+
+      def authorization_data!
+        @authorization_data = nil
+        authorization_data
+      end
+
+      def authorization_data
+        @authorization_data ||= @authorization_data_proc.call
+      end
+
+      def client!
+        @client = nil
+        client
+      end
+
+      def client
+        @client ||= client_proc.call
+      end
+
+      def retrieve_latest_cursors
+        nil
+      end
+
+      def with_auth_tokens_and_retry(&block)
+        connection_attempts = 0
+
+        begin
+          convert_transient_server_errors do
+            convert_rate_limit_errors(&block)
+          end
+        rescue ConnectorsShared::TokenRefreshFailedError => e
+          log_error('Could not refresh token, aborting')
+          raise e
+        rescue ConnectorsShared::PublishingFailedError => e
+          log_error('Could not publish, aborting')
+          raise e.reason
+        rescue ConnectorsShared::EvictionWithNoProgressError
+          log_error('Aborting job because it did not make any progress and cannot be evicted')
+          raise
+        rescue ConnectorsShared::EvictionError,
+               ConnectorsShared::ThrottlingError,
+               ConnectorsShared::JobDocumentLimitError,
+               ConnectorsShared::MonitoringError,
+               ConnectorsShared::JobInterruptedError,
+               ConnectorsShared::SecretInvalidError,
+               ConnectorsShared::InvalidIndexingConfigurationError => e
+          # Don't retry eviction, throttling, document limit, or monitoring errors, let them bubble out
+          raise
+        rescue StandardError => e
+          ConnectorsShared::ExceptionTracking.augment_exception(e)
+          connection_attempts += 1
+          if connection_attempts >= MAX_CONNECTION_ATTEMPTS
+            log_warn("Failed to connect in with_auth_tokens_and_retry Reason: #{e.class}: #{e.message} {:message_id => #{e.id}}")
+            log_warn("Retries: #{connection_attempts}/#{MAX_CONNECTION_ATTEMPTS}, giving up.")
+            ConnectorsShared::ExceptionTracking.log_exception(e)
+            raise e
+          else
+            log_warn("Failed to connect in with_auth_tokens_and_retry. Reason: #{e.class}: #{e.message} {:message_id => #{e.id}}")
+            log_warn("Retries: #{connection_attempts}/#{MAX_CONNECTION_ATTEMPTS}, trying again.")
+            retry
+          end
+        end
+      end
+
+      def yield_document_changes(modified_since: nil)
+        raise NotImplementedError
+      end
+
+      def document_changes(modified_since: nil, &block)
+        enum = nil
+        Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.documents") do
+          with_auth_tokens_and_retry do
+            Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.yield_documents") do
+              counter = 0
+              enum = Enumerator.new do |yielder|
+                yield_document_changes(:modified_since => modified_since) do |action, change, subextractors|
+                  yielder.yield action, change, subextractors
+                  counter += 1
+                  log_info("Extracted #{counter} documents so far") if counter % 100 == 0
+                end
+              end
+              enum.each(&block) if block_given?
+            end
+          end
+        end
+        enum
+      end
+
+      def yield_single_document_change(identifier: nil, &block)
+        log_debug("Extracting single document for #{identifier}") if identifier
+        convert_transient_server_errors do
+          convert_rate_limit_errors(&block)
+        end
+        monitor.note_success
+      rescue *fatal_exception_classes => e
+        ConnectorsShared::ExceptionTracking.augment_exception(e)
+        log_error("Encountered a fall-through error during extraction#{identifying_error_message(identifier)}: #{e.class}: #{e.message} {:message_id => #{e.id}}")
+        raise
+      rescue StandardError => e
+        ConnectorsShared::ExceptionTracking.augment_exception(e)
+        log_warn("Encountered error during extraction#{identifying_error_message(identifier)}: #{e.class}: #{e.message} {:message_id => #{e.id}}")
+        monitor.note_error(e, :id => e.id)
+      end
+
+      def identifying_error_message(identifier)
+        identifier.present? ? " of '#{identifier}'" : ''
+      end
+
+      def yield_deleted_ids(_ids)
+        raise NotImplementedError
+      end
+
+      def deleted_ids(ids, &block)
+        enum = nil
+        Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.deleted_ids") do
+          with_auth_tokens_and_retry do
+            Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.yield_deleted_ids") do
+              counter = 0
+              enum = Enumerator.new do |yielder|
+                yield_deleted_ids(ids) do |id|
+                  yielder.yield id
+                  counter += 1
+                  log_info("Deleted #{counter} documents so far") if counter % 100 == 0
+                end
+              end
+              enum.each(&block) if block_given?
+            end
+          end
+        end
+        enum
+      end
+
+      def yield_permissions(source_user_id)
+        # no-op for content source without DLP support
+      end
+
+      def permissions(source_user_id, &block)
+        Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.permissions") do
+          with_auth_tokens_and_retry do
+            Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.yield_permissions") do
+              yield_permissions(source_user_id) do |permissions|
+                log_info("Extracted #{permissions.size} permissions for source user #{source_user_id}")
+                block.call(permissions) if block_given?
+              end
+            end
+          end
+        end
+      end
+
+      ConnectorsShared::Logger::SUPPORTED_LOG_LEVELS.each do |log_level|
+        define_method(:"log_#{log_level}") do |message|
+          if message.kind_of?(String)
+            message = "ContentSource[#{content_source_id}, #{service_type}]: #{message}"
+          end
+          ConnectorsShared::Logger.public_send(log_level, message)
+        end
+      end
+
+      def convert_transient_server_errors
+        yield
+      rescue StandardError => e
+        raise unless transient_error?(e)
+
+        raise ConnectorsShared::TransientServerError.new(
+          "Transient error #{e.class}: #{e.message}",
+          :suspend_until => Connectors.config.fetch('transient_server_error_retry_delay_minutes').minutes.from_now,
+          :cursors => config.cursors
+        )
+      end
+
+      def transient_error?(error)
+        TRANSIENT_SERVER_ERROR_CLASSES.any? { |error_class| error.kind_of?(error_class) }
+      end
+
+      def evictable?
+        false
+      end
+
+      def cursors_modified_since_start?
+        config.cursors != original_cursors
+      end
+
+      def download_args_and_proc(id:, name:, size:, download_args:, &block)
+        [id, name, size, download_args, block]
+      end
+
+      private
+
+      def convert_rate_limit_errors
+        yield # subclasses override this with source-specific handling.
+      end
+
+      def fatal_exception_classes
+        [
+          ConnectorsShared::TokenRefreshFailedError,
+          ConnectorsShared::EvictionError,
+          ConnectorsShared::ThrottlingError,
+          ConnectorsShared::JobDocumentLimitError,
+          ConnectorsShared::MonitoringError
+        ]
+      end
+    end
+  end
+end

data/lib/connectors_sdk/base/registry.rb

@@ -0,0 +1,32 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+module ConnectorsSdk
+  module Base
+    class Factory
+      attr_reader :connectors
+
+      def initialize
+        @connectors = {}
+      end
+
+      def register(name, klass)
+        @connectors[name] = klass
+      end
+
+      def connector(name)
+        @connectors[name].new
+      end
+    end
+
+    REGISTRY = Factory.new
+
+    # loading plugins (might replace this with a directory scan and conventions on names)
+    require_relative '../share_point/http_call_wrapper'
+
+    REGISTRY.register(ConnectorsSdk::SharePoint::SERVICE_TYPE, ConnectorsSdk::SharePoint::HttpCallWrapper)
+  end
+end