connectors_sdk 8.2.0.0 → 8.3.0.0.pre.20220517T144653Z

data/lib/connectors_sdk/share_point/http_call_wrapper.rb

@@ -9,108 +9,49 @@
 require 'connectors_sdk/office365/config'
 require 'connectors_sdk/share_point/extractor'
 require 'connectors_sdk/share_point/authorization'
-require 'bson'
+require 'connectors_sdk/base/http_call_wrapper'
 
 module ConnectorsSdk
   module SharePoint
-    SERVICE_TYPE = 'share_point'
+    class HttpCallWrapper < ConnectorsSdk::Base::HttpCallWrapper
+      SERVICE_TYPE = 'share_point'
 
-    class HttpCallWrapper
-      def extractor(params)
-        cursors = params.fetch(:cursors, {}) || {}
-        features = params.fetch(:features, {}) || {}
-
-        # XXX can we cache that class across calls?
-        ConnectorsSdk::SharePoint::Extractor.new(
-          content_source_id: BSON::ObjectId.new,
-          service_type: SERVICE_TYPE,
-          authorization_data_proc: proc { { access_token: params[:access_token] } },
-          client_proc: proc { ConnectorsSdk::Office365::CustomClient.new(:access_token => params[:access_token], :cursors => cursors) },
-          config: ConnectorsSdk::Office365::Config.new(:cursors => cursors, :drive_ids => 'all', :index_permissions => params[:index_permissions] || false),
-          features: features
-        )
-      end
-
-      def document_batch(params)
-        results = []
-
-        @extractor = extractor(params)
-
-        @extractor.yield_document_changes(:break_after_page => true, :modified_since => @extractor.config.cursors['modified_since']) do |action, doc, download_args_and_proc|
-          download_obj = nil
-          if download_args_and_proc
-            download_obj = {
-              id: download_args_and_proc[0],
-              name: download_args_and_proc[1],
-              size: download_args_and_proc[2],
-              download_args: download_args_and_proc[3]
-            }
-          end
-
-          results << {
-            :action => action,
-            :document => doc,
-            :download => download_obj
-          }
-        end
-
-        results
-      rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
-        raise e.status_code == 401 ? ConnectorsShared::InvalidTokenError : e
-      end
-
-      def cursors
-        @extractor.config.cursors
+      def name
+        'SharePoint'
       end
 
-      def completed?
-        @extractor.completed
+      def service_type
+        SERVICE_TYPE
       end
 
-      def deleted(params)
-        results = []
-        extractor(params).yield_deleted_ids(params[:ids]) do |id|
-          results << id
-        end
-        results
-      rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
-        raise e.status_code == 401 ? ConnectorsShared::InvalidTokenError : e
-      end
+      private
 
-      def permissions(params)
-        extractor(params).yield_permissions(params[:user_id]) do |permissions|
-          return permissions
-        end
-      rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
-        raise e.status_code == 401 ? ConnectorsShared::InvalidTokenError : e
+      def extractor_class
+        ConnectorsSdk::SharePoint::Extractor
       end
 
-      def authorization_uri(body)
-        ConnectorsSdk::SharePoint::Authorization.authorization_uri(body)
+      def authorization
+        ConnectorsSdk::SharePoint::Authorization
       end
 
-      def access_token(params)
-        ConnectorsSdk::SharePoint::Authorization.access_token(params)
+      def client(params)
+        ConnectorsSdk::Office365::CustomClient.new(:access_token => params[:access_token], :cursors => params.fetch(:cursors, {}) || {})
       end
 
-      def refresh(params)
-        ConnectorsSdk::SharePoint::Authorization.refresh(params)
+      def custom_client_error
+        ConnectorsSdk::Office365::CustomClient::ClientError
       end
 
-      def download(params)
-        extractor(params).download(params[:meta])
-      end
-
-      def name
-        'SharePoint'
+      def config(params)
+        ConnectorsSdk::Office365::Config.new(
+          :cursors => params.fetch(:cursors, {}) || {},
+          :drive_ids => 'all',
+          :index_permissions => params[:index_permissions] || false
+        )
       end
 
-      def source_status(params)
-        client = ConnectorsSdk::Office365::CustomClient.new(:access_token => params[:access_token])
-        client.me
-        { :status => 'OK', :statusCode => 200, :message => 'Connected to SharePoint' }
-      rescue StandardError => e
-        { :status => 'FAILURE', :statusCode => e.is_a?(ConnectorsSdk::Office365::CustomClient::ClientError) ? e.status_code : 500, :message => e.message }
+      def health_check(params)
+        client(params).me
       end
     end
   end

data/lib/connectors_shared/exception_tracking.rb

@@ -6,23 +6,23 @@
 
 # frozen_string_literal: true
 
-require 'stubs/swiftype/exception_tracking' unless defined?(Rails)
 require 'bson'
 require 'connectors_shared/logger'
+require 'stubs/enterprise_search/exception_tracking'
 
 module ConnectorsShared
   class ExceptionTracking
     class << self
       def capture_message(message, context = {})
-        Swiftype::ExceptionTracking.capture_message(message, context)
+        EnterpriseSearch::ExceptionTracking.capture_message(message, context)
       end
 
       def capture_exception(exception, context = {})
-        Swiftype::ExceptionTracking.log_exception(exception, :context => context)
+        EnterpriseSearch::ExceptionTracking.log_exception(exception, :context => context)
       end
 
       def log_exception(exception, message = nil)
-        Swiftype::ExceptionTracking.log_exception(exception, message, :logger => ConnectorsShared::Logger.logger)
+        EnterpriseSearch::ExceptionTracking.log_exception(exception, message, :logger => ConnectorsShared::Logger.logger)
       end
 
       def augment_exception(exception)

data/lib/connectors_shared/extraction_utils.rb

@@ -0,0 +1,109 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'set'
+
+module ConnectorsShared
+  module ExtractionUtils
+    # A list of tags tags we want to remove before extracting content
+    NON_CONTENT_TAGS = Set.new(%w[
+      comment
+      object
+      script
+      style
+      svg
+      video
+    ]).freeze
+
+    # Tags, that generate a word/line break when rendered
+    BREAK_ELEMENTS = Set.new(%w[
+      br
+      hr
+    ]).freeze
+
+    # The character used to signal that a string has been truncated
+    OMISSION = '…'
+
+    #-------------------------------------------------------------------------------------------------
+    # Expects a Nokogiri HTML node, returns textual content from the node and all of its children
+    def self.node_descendant_text(node)
+      return '' unless node&.present?
+
+      unless node.respond_to?(:children) && node.respond_to?(:name) && node.respond_to?(:text?)
+        raise ArgumentError, "Expecting something node-like but got a #{node.class}"
+      end
+
+      to_process_stack = [node]
+      text = []
+
+      loop do
+        # Get the next node to process
+        node = to_process_stack.pop
+        break unless node
+
+        # Base cases where we append content to the text buffer
+        if node.kind_of?(String)
+          text << node unless node == ' ' && text.last == ' '
+          next
+        end
+
+        # Remove tags that do not contain any text (and which sometimes are treated as CDATA, generating garbage text in jruby)
+        next if NON_CONTENT_TAGS.include?(node.name)
+
+        # Tags, that need to be replaced by spaces according to the standards
+        if replace_with_whitespace?(node)
+          text << ' ' unless text.last == ' '
+          next
+        end
+
+        # Extract the text from all text nodes
+        if node.text?
+          content = node.content
+          text << content.squish if content
+          next
+        end
+
+        # Add spaces before all tags
+        to_process_stack << ' '
+
+        # Recursion by adding the node's children to the stack and looping
+        node.children.reverse_each { |child| to_process_stack << child }
+
+        # Add spaces after all tags
+        to_process_stack << ' '
+      end
+
+      # Remove any duplicate spaces and return the content
+      text.join.squish!
+    end
+
+    #-------------------------------------------------------------------------------------------------
+    # Returns true, if the node should be replaced with a space when extracting text from a document
+    def self.replace_with_whitespace?(node)
+      BREAK_ELEMENTS.include?(node.name)
+    end
+
+    #-------------------------------------------------------------------------------------------------
+    # Limits the size of a given string value down to a given limit (in bytes)
+    # This is heavily inspired by https://github.com/rails/rails/pull/27319/files
+    def self.limit_bytesize(string, limit)
+      return string if string.nil? || string.bytesize <= limit
+      real_limit = limit - OMISSION.bytesize
+      (+'').tap do |cut|
+        string.scan(/\X/) do |grapheme|
+          if cut.bytesize + grapheme.bytesize <= real_limit
+            cut << grapheme
+          else
+            cut << OMISSION
+            break
+          end
+        end
+      end
+    end
+  end
+end

data/lib/connectors_shared/job_status.rb

@@ -0,0 +1,18 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+module ConnectorsShared
+  class JobStatus
+    CREATED = 'created'
+    RUNNING = 'running'
+    FINISHED = 'finished'
+    FAILED = 'failed'
+
+    def self.is_valid?(status)
+      [CREATED, RUNNING, FINISHED, FAILED].include? status
+    end
+  end
+end

data/lib/connectors_shared/middleware/basic_auth.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module ConnectorsShared
+  module Middleware
+    class BasicAuth
+      AUTHORIZATION = 'Authorization'
+
+      attr_reader :basic_auth_token
+
+      def initialize(app = nil, options = {})
+        @app = app
+        @basic_auth_token = options.fetch(:basic_auth_token)
+      end
+
+      def call(env)
+        env.request_headers[AUTHORIZATION] = "Basic #{basic_auth_token}"
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/connectors_shared/middleware/bearer_auth.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module ConnectorsShared
+  module Middleware
+    class BearerAuth
+      AUTHORIZATION = 'Authorization'
+
+      attr_reader :bearer_auth_token
+
+      def initialize(app = nil, options = {})
+        @app = app
+        @bearer_auth_token = options.fetch(:bearer_auth_token)
+      end
+
+      def call(env)
+        env.request_headers[AUTHORIZATION] = "Bearer #{bearer_auth_token}"
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/connectors_shared/middleware/restrict_hostnames.rb

@@ -0,0 +1,73 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'faraday/middleware'
+require 'resolv'
+
+require 'connectors_shared/errors'
+require 'connectors_shared/logger'
+
+module ConnectorsShared
+  module Middleware
+    class RestrictHostnames < Faraday::Middleware
+      class AddressNotAllowed < ConnectorsShared::ClientError; end
+      URL_PATTERN = /\Ahttp/
+
+      attr_reader :allowed_hosts, :allowed_ips
+
+      def initialize(app = nil, options = {})
+        super(app)
+        @allowed_hosts = options[:allowed_hosts]
+        @allowed_ips = ips_from_hosts(@allowed_hosts)
+      end
+
+      def call(env)
+        raise AddressNotAllowed.new("Address not allowed for #{env[:url]}") if denied?(env)
+        @app.call(env)
+      end
+
+      private
+
+      def ips_from_hosts(hosts)
+        hosts&.flat_map do |host|
+          if URL_PATTERN.match(host)
+            lookup_ips(URI.parse(host).host)
+          elsif Resolv::IPv4::Regex.match(host) || Resolv::IPv6::Regex.match(host)
+            IPAddr.new(host)
+          else
+            lookup_ips(host)
+          end
+        end || []
+      end
+
+      def denied?(env)
+        requested_ips = lookup_ips(env[:url].hostname)
+        no_match = requested_ips.all? { |ip| !@allowed_ips.include?(ip) }
+        return false unless no_match
+        ConnectorsShared::Logger.warn("Requested url #{env[:url]} with resolved ip addresses #{requested_ips} does not match " \
+                                  "allowed hosts #{@allowed_hosts} with resolved ip addresses #{@allowed_ips}. Retrying.")
+        @allowed_ips = ips_from_hosts(@allowed_hosts) # maybe the IP has changed for an allowed host. Re-do allowed_hosts DNS lookup
+        no_match = requested_ips.all? { |ip| !@allowed_ips.include?(ip) }
+        ConnectorsShared::Logger.error("Requested url #{env[:url]} with resolved ip addresses #{requested_ips} does not match " \
+                                  "allowed hosts #{@allowed_hosts} with resolved ip addresses #{@allowed_ips}") if no_match
+        no_match
+      end
+
+      def lookup_ips(hostname)
+        addr_infos(hostname).map { |a| IPAddr.new(a.ip_address) }
+      end
+
+      def addr_infos(hostname)
+        Addrinfo.getaddrinfo(hostname, nil, :UNSPEC, :STREAM)
+      rescue SocketError
+        # In case of invalid hostname, return an empty list of addresses
+        []
+      end
+    end
+  end
+end

data/lib/connectors_shared/monitor.rb

@@ -8,7 +8,7 @@
 
 require 'connectors_shared/errors'
 require 'stubs/app_config' unless defined?(Rails)
-require 'stubs/swiftype/exception_tracking' unless defined?(Rails)
+require 'stubs/enterprise_search/exception_tracking' unless defined?(Rails)
 
 module ConnectorsShared
   class Monitor
@@ -44,8 +44,8 @@ module ConnectorsShared
     end
 
     def note_error(error, id: Time.now.to_i)
-      stack_trace = Swiftype::ExceptionTracking.generate_stack_trace(error)
-      error_message = Swiftype::ExceptionTracking.generate_error_message(error, nil, nil)
+      stack_trace = EnterpriseSearch::ExceptionTracking.generate_stack_trace(error)
+      error_message = EnterpriseSearch::ExceptionTracking.generate_error_message(error, nil, nil)
       @connector.log_debug("Message id: #{id} - #{error_message}\n#{stack_trace}")
       @total_error_count += 1
       @consecutive_error_count += 1

data/lib/connectors_shared.rb

@@ -8,5 +8,6 @@ require 'connectors_shared/constants'
 require 'connectors_shared/errors'
 require 'connectors_shared/exception_tracking'
 require 'connectors_shared/extension_mapping_util'
+require 'connectors_shared/job_status'
 require 'connectors_shared/logger'
 require 'connectors_shared/monitor'

data/lib/stubs/enterprise_search/exception_tracking.rb

@@ -0,0 +1,43 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+module EnterpriseSearch
+  class ExceptionTracking
+    def self.capture_message(message, context = {})
+      AppConfig.connectors_logger.error { "Error: #{message}. Context: #{context.inspect}" }
+
+      # When the method is called from a rescue block, our return value may leak outside of its
+      # intended scope, so let's explicitly return nil here to be safe.
+      nil
+    end
+
+    def self.log_exception(exception, message = nil, context: nil, logger: AppConfig.connectors_logger)
+      logger.error { message } if message
+      logger.error { generate_stack_trace(exception) }
+      logger.error { "Context: #{context.inspect}" } if context
+    end
+
+    def self.generate_error_message(exception, message, context)
+      context = { :message_id => exception.id }.merge(context || {}) if exception.respond_to?(:id)
+      context_message = context && "Context: #{context.inspect}"
+      ['Exception', message, exception.class.to_s, exception.message, context_message]
+        .compact
+        .map { |part| part.to_s.dup.force_encoding('UTF-8') }
+        .join(': ')
+    end
+
+    def self.generate_stack_trace(exception)
+      full_message = exception.full_message
+
+      cause = exception
+      while cause.cause != cause && (cause = cause.cause)
+        full_message << "Cause:\n#{cause.full_message}"
+      end
+
+      full_message.dup.force_encoding('UTF-8')
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_sdk
 version: !ruby/object:Gem::Version
-  version: 8.2.0.0
+  version: 8.3.0.0.pre.20220517T144653Z
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-14 00:00:00.000000000 Z
+date: 2022-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -61,11 +61,23 @@ files:
 - LICENSE
 - NOTICE.txt
 - lib/connectors_sdk.rb
+- lib/connectors_sdk/atlassian/config.rb
+- lib/connectors_sdk/atlassian/custom_client.rb
 - lib/connectors_sdk/base/adapter.rb
+- lib/connectors_sdk/base/authorization.rb
 - lib/connectors_sdk/base/config.rb
 - lib/connectors_sdk/base/custom_client.rb
 - lib/connectors_sdk/base/extractor.rb
+- lib/connectors_sdk/base/http_call_wrapper.rb
 - lib/connectors_sdk/base/registry.rb
+- lib/connectors_sdk/confluence/adapter.rb
+- lib/connectors_sdk/confluence/custom_client.rb
+- lib/connectors_sdk/confluence/extractor.rb
+- lib/connectors_sdk/confluence_cloud/authorization.rb
+- lib/connectors_sdk/confluence_cloud/custom_client.rb
+- lib/connectors_sdk/confluence_cloud/extractor.rb
+- lib/connectors_sdk/confluence_cloud/http_call_wrapper.rb
+- lib/connectors_sdk/helpers/atlassian_time_formatter.rb
 - lib/connectors_sdk/office365/adapter.rb
 - lib/connectors_sdk/office365/config.rb
 - lib/connectors_sdk/office365/custom_client.rb
@@ -79,13 +91,19 @@ files:
 - lib/connectors_shared/errors.rb
 - lib/connectors_shared/exception_tracking.rb
 - lib/connectors_shared/extension_mapping_util.rb
+- lib/connectors_shared/extraction_utils.rb
+- lib/connectors_shared/job_status.rb
 - lib/connectors_shared/logger.rb
+- lib/connectors_shared/middleware/basic_auth.rb
+- lib/connectors_shared/middleware/bearer_auth.rb
+- lib/connectors_shared/middleware/restrict_hostnames.rb
 - lib/connectors_shared/monitor.rb
+- lib/stubs/enterprise_search/exception_tracking.rb
 homepage: https://github.com/elastic/connectors
 licenses:
 - Elastic-2.0
 metadata:
-  revision: 8630b57d72a5cef405ee3c5b8430b09e7a2f388d
+  revision: 9f25f35e17ffb36dfda754d657794ed9b5d2d75a
   repository: git@github.com:elastic/connectors.git
 post_install_message:
 rdoc_options: []
@@ -98,9 +116,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3.1
 signing_key: