connectors_sdk 8.3.0.0.pre.20220414T060419Z → 8.3.0.0.pre.20220510T144908Z

data/lib/connectors_shared/middleware/restrict_hostnames.rb

@@ -0,0 +1,73 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'faraday/middleware'
+require 'resolv'
+
+require 'connectors_shared/errors'
+require 'connectors_shared/logger'
+
+module ConnectorsShared
+  module Middleware
+    class RestrictHostnames < Faraday::Middleware
+      class AddressNotAllowed < ConnectorsShared::ClientError; end
+      URL_PATTERN = /\Ahttp/
+
+      attr_reader :allowed_hosts, :allowed_ips
+
+      def initialize(app = nil, options = {})
+        super(app)
+        @allowed_hosts = options[:allowed_hosts]
+        @allowed_ips = ips_from_hosts(@allowed_hosts)
+      end
+
+      def call(env)
+        raise AddressNotAllowed.new("Address not allowed for #{env[:url]}") if denied?(env)
+        @app.call(env)
+      end
+
+      private
+
+      def ips_from_hosts(hosts)
+        hosts&.flat_map do |host|
+          if URL_PATTERN.match(host)
+            lookup_ips(URI.parse(host).host)
+          elsif Resolv::IPv4::Regex.match(host) || Resolv::IPv6::Regex.match(host)
+            IPAddr.new(host)
+          else
+            lookup_ips(host)
+          end
+        end || []
+      end
+
+      def denied?(env)
+        requested_ips = lookup_ips(env[:url].hostname)
+        no_match = requested_ips.all? { |ip| !@allowed_ips.include?(ip) }
+        return false unless no_match
+        ConnectorsShared::Logger.warn("Requested url #{env[:url]} with resolved ip addresses #{requested_ips} does not match " \
+                                  "allowed hosts #{@allowed_hosts} with resolved ip addresses #{@allowed_ips}. Retrying.")
+        @allowed_ips = ips_from_hosts(@allowed_hosts) # maybe the IP has changed for an allowed host. Re-do allowed_hosts DNS lookup
+        no_match = requested_ips.all? { |ip| !@allowed_ips.include?(ip) }
+        ConnectorsShared::Logger.error("Requested url #{env[:url]} with resolved ip addresses #{requested_ips} does not match " \
+                                  "allowed hosts #{@allowed_hosts} with resolved ip addresses #{@allowed_ips}") if no_match
+        no_match
+      end
+
+      def lookup_ips(hostname)
+        addr_infos(hostname).map { |a| IPAddr.new(a.ip_address) }
+      end
+
+      def addr_infos(hostname)
+        Addrinfo.getaddrinfo(hostname, nil, :UNSPEC, :STREAM)
+      rescue SocketError
+        # In case of invalid hostname, return an empty list of addresses
+        []
+      end
+    end
+  end
+end

data/lib/connectors_shared/monitor.rb

@@ -8,7 +8,7 @@
 
 require 'connectors_shared/errors'
 require 'stubs/app_config' unless defined?(Rails)
-require 'stubs/swiftype/exception_tracking' unless defined?(Rails)
+require 'stubs/enterprise_search/exception_tracking' unless defined?(Rails)
 
 module ConnectorsShared
   class Monitor
@@ -44,8 +44,8 @@ module ConnectorsShared
     end
 
     def note_error(error, id: Time.now.to_i)
-      stack_trace = Swiftype::ExceptionTracking.generate_stack_trace(error)
-      error_message = Swiftype::ExceptionTracking.generate_error_message(error, nil, nil)
+      stack_trace = EnterpriseSearch::ExceptionTracking.generate_stack_trace(error)
+      error_message = EnterpriseSearch::ExceptionTracking.generate_error_message(error, nil, nil)
       @connector.log_debug("Message id: #{id} - #{error_message}\n#{stack_trace}")
       @total_error_count += 1
       @consecutive_error_count += 1

data/lib/stubs/enterprise_search/exception_tracking.rb

@@ -0,0 +1,43 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+module EnterpriseSearch
+  class ExceptionTracking
+    def self.capture_message(message, context = {})
+      AppConfig.connectors_logger.error { "Error: #{message}. Context: #{context.inspect}" }
+
+      # When the method is called from a rescue block, our return value may leak outside of its
+      # intended scope, so let's explicitly return nil here to be safe.
+      nil
+    end
+
+    def self.log_exception(exception, message = nil, context: nil, logger: AppConfig.connectors_logger)
+      logger.error { message } if message
+      logger.error { generate_stack_trace(exception) }
+      logger.error { "Context: #{context.inspect}" } if context
+    end
+
+    def self.generate_error_message(exception, message, context)
+      context = { :message_id => exception.id }.merge(context || {}) if exception.respond_to?(:id)
+      context_message = context && "Context: #{context.inspect}"
+      ['Exception', message, exception.class.to_s, exception.message, context_message]
+        .compact
+        .map { |part| part.to_s.dup.force_encoding('UTF-8') }
+        .join(': ')
+    end
+
+    def self.generate_stack_trace(exception)
+      full_message = exception.full_message
+
+      cause = exception
+      while cause.cause != cause && (cause = cause.cause)
+        full_message << "Cause:\n#{cause.full_message}"
+      end
+
+      full_message.dup.force_encoding('UTF-8')
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_sdk
 version: !ruby/object:Gem::Version
-  version: 8.3.0.0.pre.20220414T060419Z
+  version: 8.3.0.0.pre.20220510T144908Z
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-14 00:00:00.000000000 Z
+date: 2022-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -61,19 +61,27 @@ files:
 - LICENSE
 - NOTICE.txt
 - lib/connectors_sdk.rb
-- lib/connectors_sdk/base/.config.rb.un~
-- lib/connectors_sdk/base/.connectors.rb.un~
-- lib/connectors_sdk/base/.registry.rb.un~
+- lib/connectors_sdk/atlassian/config.rb
+- lib/connectors_sdk/atlassian/custom_client.rb
 - lib/connectors_sdk/base/adapter.rb
+- lib/connectors_sdk/base/authorization.rb
 - lib/connectors_sdk/base/config.rb
 - lib/connectors_sdk/base/custom_client.rb
 - lib/connectors_sdk/base/extractor.rb
+- lib/connectors_sdk/base/http_call_wrapper.rb
 - lib/connectors_sdk/base/registry.rb
+- lib/connectors_sdk/confluence/adapter.rb
+- lib/connectors_sdk/confluence/custom_client.rb
+- lib/connectors_sdk/confluence/extractor.rb
+- lib/connectors_sdk/confluence_cloud/authorization.rb
+- lib/connectors_sdk/confluence_cloud/custom_client.rb
+- lib/connectors_sdk/confluence_cloud/extractor.rb
+- lib/connectors_sdk/confluence_cloud/http_call_wrapper.rb
+- lib/connectors_sdk/helpers/atlassian_time_formatter.rb
 - lib/connectors_sdk/office365/adapter.rb
 - lib/connectors_sdk/office365/config.rb
 - lib/connectors_sdk/office365/custom_client.rb
 - lib/connectors_sdk/office365/extractor.rb
-- lib/connectors_sdk/share_point/.http_call_wrapper.rb.un~
 - lib/connectors_sdk/share_point/adapter.rb
 - lib/connectors_sdk/share_point/authorization.rb
 - lib/connectors_sdk/share_point/extractor.rb
@@ -83,14 +91,19 @@ files:
 - lib/connectors_shared/errors.rb
 - lib/connectors_shared/exception_tracking.rb
 - lib/connectors_shared/extension_mapping_util.rb
+- lib/connectors_shared/extraction_utils.rb
 - lib/connectors_shared/logger.rb
+- lib/connectors_shared/middleware/basic_auth.rb
+- lib/connectors_shared/middleware/bearer_auth.rb
+- lib/connectors_shared/middleware/restrict_hostnames.rb
 - lib/connectors_shared/monitor.rb
+- lib/stubs/enterprise_search/exception_tracking.rb
 homepage: https://github.com/elastic/connectors
 licenses:
 - Elastic-2.0
 metadata:
-  revision: 427fad57fc69a8107c127ecbf99c6a5e803ea90c
-  repository: https://github.com/elastic/connectors.git
+  revision: fb1187beef857b555633e1804eef3ed5e586091d
+  repository: git@github.com:elastic/connectors.git
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -106,8 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.7
+rubygems_version: 3.0.3.1
 signing_key:
 specification_version: 4
 summary: Gem containing apis used by Enterprise Search and implementations of Connectors