connectors_sdk 8.3.0.0.pre.20220414T060419Z → 8.3.0.0

data/lib/connectors_sdk/share_point/authorization.rb

@@ -6,70 +6,12 @@
 
 # frozen_string_literal: true
 
-require 'connectors_shared'
-require 'signet'
-require 'signet/oauth_2'
-require 'signet/oauth_2/client'
+require 'connectors_sdk/base/authorization'
 
 module ConnectorsSdk
   module SharePoint
-    class Authorization
+    class Authorization < ConnectorsSdk::Base::Authorization
       class << self
-        def authorization_url
-          'https://login.microsoftonline.com/common/oauth2/v2.0/authorize'
-        end
-
-        def token_credential_uri
-          'https://login.microsoftonline.com/common/oauth2/v2.0/token'
-        end
-
-        def authorization_uri(params)
-          missing = missing_fields(params, %w[client_id])
-          unless missing.blank?
-            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
-          end
-
-          params[:response_type] = 'code'
-          params[:additional_parameters] = { :prompt => 'consent' }
-          client = oauth_client(params)
-          client.authorization_uri.to_s
-        end
-
-        def access_token(params)
-          missing = missing_fields(params, %w[client_id client_secret code redirect_uri])
-          unless missing.blank?
-            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
-          end
-
-          params[:grant_type] = 'authorization_code'
-          client = oauth_client(params)
-          client.fetch_access_token
-        end
-
-        def refresh(params)
-          missing = missing_fields(params, %w[client_id client_secret refresh_token])
-          unless missing.blank?
-            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
-          end
-
-          params[:grant_type] = 'refresh_token'
-          client = oauth_client(params)
-          client.refresh!
-        rescue StandardError => e
-          ConnectorsShared::ExceptionTracking.log_exception(e)
-          raise ConnectorsShared::TokenRefreshFailedError
-        end
-
-        def oauth_client(params)
-          options = params.merge(
-            :authorization_uri => authorization_url,
-            :token_credential_uri => token_credential_uri,
-            :scope => oauth_scope
-          )
-          options[:state] = JSON.dump(options[:state]) if options[:state]
-          Signet::OAuth2::Client.new(options)
-        end
-
         def oauth_scope
           %w[
             User.ReadBasic.All
@@ -82,8 +24,18 @@ module ConnectorsSdk
           ]
         end
 
-        def missing_fields(params, required = [])
-          Array.wrap(required).select { |field| params[field.to_sym].nil? }
+        private
+
+        def authorization_url
+          'https://login.microsoftonline.com/common/oauth2/v2.0/authorize'
+        end
+
+        def token_credential_uri
+          'https://login.microsoftonline.com/common/oauth2/v2.0/token'
+        end
+
+        def additional_parameters
+          { :prompt => 'consent' }
         end
       end
     end

data/lib/connectors_sdk/share_point/connector.rb

@@ -0,0 +1,82 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/connector'
+require 'connectors_sdk/office365/config'
+require 'connectors_sdk/share_point/extractor'
+require 'connectors_sdk/share_point/authorization'
+
+module ConnectorsSdk
+  module SharePoint
+    class Connector < ConnectorsSdk::Base::Connector
+      SERVICE_TYPE = 'share_point'
+
+      def compare_secrets(params)
+        missing_secrets?(params)
+
+        previous_user = client(:access_token => params[:other_secret][:access_token]).me
+        equivalent = previous_user.nil? ? false : previous_user.id == client(:access_token => params[:secret][:access_token]).me&.id
+
+        {
+          :equivalent => equivalent
+        }
+      end
+
+      def display_name
+        'SharePoint Online'
+      end
+
+      def connection_requires_redirect
+        true
+      end
+
+      def configurable_fields
+        [
+          {
+            'key' => 'client_id',
+            'label' => 'Client ID'
+          },
+          {
+            'key' => 'client_secret',
+            'label' => 'Client Secret'
+          },
+        ]
+      end
+
+      private
+
+      def extractor_class
+        ConnectorsSdk::SharePoint::Extractor
+      end
+
+      def authorization
+        ConnectorsSdk::SharePoint::Authorization
+      end
+
+      def client(params)
+        ConnectorsSdk::Office365::CustomClient.new(:access_token => params[:access_token], :cursors => params.fetch(:cursors, {}) || {})
+      end
+
+      def custom_client_error
+        ConnectorsSdk::Office365::CustomClient::ClientError
+      end
+
+      def config(params)
+        ConnectorsSdk::Office365::Config.new(
+          :cursors => params.fetch(:cursors, {}) || {},
+          :drive_ids => 'all',
+          :index_permissions => params[:index_permissions] || false
+        )
+      end
+
+      def health_check(params)
+        client(params).me
+      end
+    end
+  end
+end

data/lib/connectors_sdk/share_point/extractor.rb

@@ -15,8 +15,8 @@ module ConnectorsSdk
 
       private
 
-      def convert_id_to_fp_id(id)
-        ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+      def convert_id_to_es_id(id)
+        ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
       end
 
       def adapter

data/lib/connectors_sdk/stub_connector/connector.rb

@@ -0,0 +1,62 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/connector'
+
+module ConnectorsSdk
+  module StubConnector
+    class Connector < ConnectorsSdk::Base::Connector
+      SERVICE_TYPE = 'stub_connector'
+
+      def display_name
+        'Stub Connector'
+      end
+
+      def configurable_fields
+        [
+          {
+            'key' => 'third_party_url',
+            'label' => 'Third Party URL'
+          },
+          {
+            'key' => 'third_party_api_key',
+            'label' => 'Third Party API Key'
+          }
+        ]
+      end
+
+      def health_check(_params)
+        true
+      end
+
+      def document_batch(_params)
+        results = 30.times.map do |i|
+          {
+            :action => :create_or_update,
+            :document => {
+              :id => "document_#{i}",
+              :type => 'document',
+              :body => "contents for document number: #{i}"
+            },
+            :download => nil
+          }
+        end
+
+        [results, {}, true]
+      end
+
+      def deleted(_params)
+        []
+      end
+
+      def permissions(_params)
+        []
+      end
+    end
+  end
+end

data/lib/connectors_shared/constants.rb

@@ -10,5 +10,17 @@ module ConnectorsShared
     SUBEXTRACTOR_RESERVED_FIELDS = %w[_subextracted_as_of _subextracted_version].freeze
     ALLOW_FIELD = '_allow_permissions'.freeze
     DENY_FIELD = '_deny_permissions'.freeze
+
+    # The following section reads as following:
+    # The job will extract documents until the job queue size will reach
+    # JOB_QUEUE_SIZE_IDLE_THRESHOLD items. After that, the job will attempt to sleep
+    # for IDLE_SLEEP_TIME seconds and check the queue size again. If the queue is still
+    # full, it will sleep for maximum MAX_IDDLE_ATTEMPTS times, and if the queue is still
+    # full, then job will be terminated.
+    JOB_QUEUE_SIZE_IDLE_THRESHOLD = 500 # How many documents the job queue stores until it sleeps
+    IDLE_SLEEP_TIME = 10 # For how long job queue will sleep before checking the queue size again
+    MAX_IDLE_ATTEMPTS = 30 # How many consecutive times job will try to sleep until it's destroyed
+
+    STALE_JOB_TIMEOUT = 60 * 30 # Time in seconds after which the job will be cleaned up if the job is considered stuck
   end
 end

data/lib/connectors_shared/exception_tracking.rb

@@ -6,23 +6,23 @@
 
 # frozen_string_literal: true
 
-require 'stubs/swiftype/exception_tracking' unless defined?(Rails)
 require 'bson'
 require 'connectors_shared/logger'
+require 'stubs/enterprise_search/exception_tracking'
 
 module ConnectorsShared
   class ExceptionTracking
     class << self
       def capture_message(message, context = {})
-        Swiftype::ExceptionTracking.capture_message(message, context)
+        EnterpriseSearch::ExceptionTracking.capture_message(message, context)
       end
 
       def capture_exception(exception, context = {})
-        Swiftype::ExceptionTracking.log_exception(exception, :context => context)
+        EnterpriseSearch::ExceptionTracking.log_exception(exception, :context => context)
       end
 
       def log_exception(exception, message = nil)
-        Swiftype::ExceptionTracking.log_exception(exception, message, :logger => ConnectorsShared::Logger.logger)
+        EnterpriseSearch::ExceptionTracking.log_exception(exception, message, :logger => ConnectorsShared::Logger.logger)
       end
 
       def augment_exception(exception)

data/lib/connectors_shared/extraction_utils.rb

@@ -0,0 +1,109 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'set'
+
+module ConnectorsShared
+  module ExtractionUtils
+    # A list of tags tags we want to remove before extracting content
+    NON_CONTENT_TAGS = Set.new(%w[
+      comment
+      object
+      script
+      style
+      svg
+      video
+    ]).freeze
+
+    # Tags, that generate a word/line break when rendered
+    BREAK_ELEMENTS = Set.new(%w[
+      br
+      hr
+    ]).freeze
+
+    # The character used to signal that a string has been truncated
+    OMISSION = '…'
+
+    #-------------------------------------------------------------------------------------------------
+    # Expects a Nokogiri HTML node, returns textual content from the node and all of its children
+    def self.node_descendant_text(node)
+      return '' unless node&.present?
+
+      unless node.respond_to?(:children) && node.respond_to?(:name) && node.respond_to?(:text?)
+        raise ArgumentError, "Expecting something node-like but got a #{node.class}"
+      end
+
+      to_process_stack = [node]
+      text = []
+
+      loop do
+        # Get the next node to process
+        node = to_process_stack.pop
+        break unless node
+
+        # Base cases where we append content to the text buffer
+        if node.kind_of?(String)
+          text << node unless node == ' ' && text.last == ' '
+          next
+        end
+
+        # Remove tags that do not contain any text (and which sometimes are treated as CDATA, generating garbage text in jruby)
+        next if NON_CONTENT_TAGS.include?(node.name)
+
+        # Tags, that need to be replaced by spaces according to the standards
+        if replace_with_whitespace?(node)
+          text << ' ' unless text.last == ' '
+          next
+        end
+
+        # Extract the text from all text nodes
+        if node.text?
+          content = node.content
+          text << content.squish if content
+          next
+        end
+
+        # Add spaces before all tags
+        to_process_stack << ' '
+
+        # Recursion by adding the node's children to the stack and looping
+        node.children.reverse_each { |child| to_process_stack << child }
+
+        # Add spaces after all tags
+        to_process_stack << ' '
+      end
+
+      # Remove any duplicate spaces and return the content
+      text.join.squish!
+    end
+
+    #-------------------------------------------------------------------------------------------------
+    # Returns true, if the node should be replaced with a space when extracting text from a document
+    def self.replace_with_whitespace?(node)
+      BREAK_ELEMENTS.include?(node.name)
+    end
+
+    #-------------------------------------------------------------------------------------------------
+    # Limits the size of a given string value down to a given limit (in bytes)
+    # This is heavily inspired by https://github.com/rails/rails/pull/27319/files
+    def self.limit_bytesize(string, limit)
+      return string if string.nil? || string.bytesize <= limit
+      real_limit = limit - OMISSION.bytesize
+      (+'').tap do |cut|
+        string.scan(/\X/) do |grapheme|
+          if cut.bytesize + grapheme.bytesize <= real_limit
+            cut << grapheme
+          else
+            cut << OMISSION
+            break
+          end
+        end
+      end
+    end
+  end
+end

data/lib/connectors_shared/job_status.rb

@@ -0,0 +1,18 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+module ConnectorsShared
+  class JobStatus
+    CREATED = 'created'
+    RUNNING = 'running'
+    FINISHED = 'finished'
+    FAILED = 'failed'
+
+    def self.is_valid?(status)
+      [CREATED, RUNNING, FINISHED, FAILED].include? status
+    end
+  end
+end

data/lib/connectors_shared/middleware/basic_auth.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module ConnectorsShared
+  module Middleware
+    class BasicAuth
+      AUTHORIZATION = 'Authorization'
+
+      attr_reader :basic_auth_token
+
+      def initialize(app = nil, options = {})
+        @app = app
+        @basic_auth_token = options.fetch(:basic_auth_token)
+      end
+
+      def call(env)
+        env.request_headers[AUTHORIZATION] = "Basic #{basic_auth_token}"
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/connectors_shared/middleware/bearer_auth.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module ConnectorsShared
+  module Middleware
+    class BearerAuth
+      AUTHORIZATION = 'Authorization'
+
+      attr_reader :bearer_auth_token
+
+      def initialize(app = nil, options = {})
+        @app = app
+        @bearer_auth_token = options.fetch(:bearer_auth_token)
+      end
+
+      def call(env)
+        env.request_headers[AUTHORIZATION] = "Bearer #{bearer_auth_token}"
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/connectors_shared/middleware/restrict_hostnames.rb

@@ -0,0 +1,73 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'faraday/middleware'
+require 'resolv'
+
+require 'connectors_shared/errors'
+require 'connectors_shared/logger'
+
+module ConnectorsShared
+  module Middleware
+    class RestrictHostnames < Faraday::Middleware
+      class AddressNotAllowed < ConnectorsShared::ClientError; end
+      URL_PATTERN = /\Ahttp/
+
+      attr_reader :allowed_hosts, :allowed_ips
+
+      def initialize(app = nil, options = {})
+        super(app)
+        @allowed_hosts = options[:allowed_hosts]
+        @allowed_ips = ips_from_hosts(@allowed_hosts)
+      end
+
+      def call(env)
+        raise AddressNotAllowed.new("Address not allowed for #{env[:url]}") if denied?(env)
+        @app.call(env)
+      end
+
+      private
+
+      def ips_from_hosts(hosts)
+        hosts&.flat_map do |host|
+          if URL_PATTERN.match(host)
+            lookup_ips(Addressable::URI.parse(host).hostname)
+          elsif Resolv::IPv4::Regex.match(host) || Resolv::IPv6::Regex.match(host)
+            IPAddr.new(host)
+          else
+            lookup_ips(host)
+          end
+        end || []
+      end
+
+      def denied?(env)
+        requested_ips = lookup_ips(env[:url].hostname)
+        no_match = requested_ips.all? { |ip| !@allowed_ips.include?(ip) }
+        return false unless no_match
+        ConnectorsShared::Logger.warn("Requested url #{env[:url]} with resolved ip addresses #{requested_ips} does not match " \
+                                  "allowed hosts #{@allowed_hosts} with resolved ip addresses #{@allowed_ips}. Retrying.")
+        @allowed_ips = ips_from_hosts(@allowed_hosts) # maybe the IP has changed for an allowed host. Re-do allowed_hosts DNS lookup
+        no_match = requested_ips.all? { |ip| !@allowed_ips.include?(ip) }
+        ConnectorsShared::Logger.error("Requested url #{env[:url]} with resolved ip addresses #{requested_ips} does not match " \
+                                  "allowed hosts #{@allowed_hosts} with resolved ip addresses #{@allowed_ips}") if no_match
+        no_match
+      end
+
+      def lookup_ips(hostname)
+        addr_infos(hostname).map { |a| IPAddr.new(a.ip_address) }
+      end
+
+      def addr_infos(hostname)
+        Addrinfo.getaddrinfo(hostname, nil, :UNSPEC, :STREAM)
+      rescue SocketError
+        # In case of invalid hostname, return an empty list of addresses
+        []
+      end
+    end
+  end
+end

data/lib/connectors_shared/monitor.rb

@@ -8,7 +8,7 @@
 
 require 'connectors_shared/errors'
 require 'stubs/app_config' unless defined?(Rails)
-require 'stubs/swiftype/exception_tracking' unless defined?(Rails)
+require 'stubs/enterprise_search/exception_tracking' unless defined?(Rails)
 
 module ConnectorsShared
   class Monitor
@@ -44,8 +44,8 @@ module ConnectorsShared
     end
 
     def note_error(error, id: Time.now.to_i)
-      stack_trace = Swiftype::ExceptionTracking.generate_stack_trace(error)
-      error_message = Swiftype::ExceptionTracking.generate_error_message(error, nil, nil)
+      stack_trace = EnterpriseSearch::ExceptionTracking.generate_stack_trace(error)
+      error_message = EnterpriseSearch::ExceptionTracking.generate_error_message(error, nil, nil)
       @connector.log_debug("Message id: #{id} - #{error_message}\n#{stack_trace}")
       @total_error_count += 1
       @consecutive_error_count += 1

data/lib/connectors_shared.rb

@@ -8,5 +8,6 @@ require 'connectors_shared/constants'
 require 'connectors_shared/errors'
 require 'connectors_shared/exception_tracking'
 require 'connectors_shared/extension_mapping_util'
+require 'connectors_shared/job_status'
 require 'connectors_shared/logger'
 require 'connectors_shared/monitor'

data/lib/stubs/enterprise_search/exception_tracking.rb

@@ -0,0 +1,43 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+module EnterpriseSearch
+  class ExceptionTracking
+    def self.capture_message(message, context = {})
+      AppConfig.connectors_logger.error { "Error: #{message}. Context: #{context.inspect}" }
+
+      # When the method is called from a rescue block, our return value may leak outside of its
+      # intended scope, so let's explicitly return nil here to be safe.
+      nil
+    end
+
+    def self.log_exception(exception, message = nil, context: nil, logger: AppConfig.connectors_logger)
+      logger.error { message } if message
+      logger.error { generate_stack_trace(exception) }
+      logger.error { "Context: #{context.inspect}" } if context
+    end
+
+    def self.generate_error_message(exception, message, context)
+      context = { :message_id => exception.id }.merge(context || {}) if exception.respond_to?(:id)
+      context_message = context && "Context: #{context.inspect}"
+      ['Exception', message, exception.class.to_s, exception.message, context_message]
+        .compact
+        .map { |part| part.to_s.dup.force_encoding('UTF-8') }
+        .join(': ')
+    end
+
+    def self.generate_stack_trace(exception)
+      full_message = exception.full_message
+
+      cause = exception
+      while cause.cause != cause && (cause = cause.cause)
+        full_message << "Cause:\n#{cause.full_message}"
+      end
+
+      full_message.dup.force_encoding('UTF-8')
+    end
+  end
+end