connectors_sdk 8.3.0.0.pre.20220517T144653Z → 8.3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b86ca5e489e3cef3b9f2c04a462baf71d6b43805731b0cb52ba2f56f5078d6d3
-  data.tar.gz: 044e860f11163e82c63f66276c3d5628b761c5dfcc5168fc8f80b63ca87d19f0
+  metadata.gz: 21bd11a44ed7c45fa04b218346fda20d946b97ec8f7025c2cfb8c8f92321d6a3
+  data.tar.gz: b388f972974f32c59d719b809975229125e293045dd35559546fefe5485e1fc3
 SHA512:
-  metadata.gz: 528fa5260cf80a3ebb918478e1be2e7cac1668588853ad72b1259095086090c15a0cbc028523b1de8b775b3b64ed7427c6fab7ffccc562d559c8164456b84c4b
-  data.tar.gz: a58d353e2b48ffda33aa287d0fa6bb1400c531aae2794995ff3b644b779426d10bb3efc8735d8e07c353fae64bc0b3b0af58427d989f499210a19244b23c1a35
+  metadata.gz: 88aa76506d6f2af049ea2a2ff54797270cef692319dd9ca647d33903a9fb3bb8bb274d9d99a3f94fe4d54cfce1722b31d63203cd157b627ccf3b2ca541b63523
+  data.tar.gz: 3faf8e16b18c0abd4f13fa988de8a6bb03e95bce5738389f132748fc17a230a9e516eb8b9f898d60aeb8a869f1ad1dadbb8606178e01be7d9ccc63ca98dcf681

data/lib/connectors_sdk/base/{http_call_wrapper.rb → connector.rb}

@@ -10,13 +10,25 @@ require 'bson'
 
 module ConnectorsSdk
   module Base
-    class HttpCallWrapper
+    class Connector
       def extractor(params)
+        content_source_id = params.fetch(:content_source_id)
+        secret_storage = params[:secret_storage]
+
         extractor_class.new(
-          content_source_id: params[:content_source_id] || "GENERATED-#{BSON::ObjectId.new}",
+          content_source_id: content_source_id || "GENERATED-#{BSON::ObjectId.new}",
           service_type: service_type,
-          authorization_data_proc: proc { { access_token: params[:access_token] } },
-          client_proc: proc { client(params) },
+          authorization_data_proc: proc do
+            secret = secret_storage.fetch_secret(content_source_id)
+            {
+              access_token: secret[:access_token]
+            }
+          end,
+          client_proc: proc {
+            secret = secret_storage.fetch_secret(content_source_id)
+            params[:access_token] = secret[:access_token]
+            client(params)
+          },
           config: config(params),
           features: params.fetch(:features, {}) || {}
         )
@@ -68,6 +80,10 @@ module ConnectorsSdk
         end
       end
 
+      def download(params)
+        extractor(params).download(params[:meta])
+      end
+
       def authorization_uri(params)
         authorization.authorization_uri(params)
       end
@@ -80,18 +96,18 @@ module ConnectorsSdk
         authorization.refresh(params)
       end
 
-      def download(params)
-        extractor(params).download(params[:meta])
-      end
-
       def source_status(params)
         health_check(params)
-        { :status => 'OK', :statusCode => 200, :message => "Connected to #{name}" }
+        { :status => 'OK', :statusCode => 200, :message => "Connected to #{display_name}" }
       rescue StandardError => e
         { :status => 'FAILURE', :statusCode => e.is_a?(custom_client_error) ? e.status_code : 500, :message => e.message }
       end
 
-      def name
+      def compare_secrets(*)
+        raise 'Not implemented for this connector'
+      end
+
+      def display_name
         raise 'Not implemented for this connector'
       end
 
@@ -99,6 +115,14 @@ module ConnectorsSdk
         self.class::SERVICE_TYPE
       end
 
+      def connection_requires_redirect
+        false
+      end
+
+      def configurable_fields
+        []
+      end
+
       private
 
       def convert_third_party_errors
@@ -130,6 +154,13 @@ module ConnectorsSdk
       def health_check(*)
         raise 'Not implemented for this connector'
       end
+
+      def missing_secrets?(params)
+        missing = %w[secret other_secret].select { |field| params[field.to_sym].nil? }
+        unless missing.blank?
+          raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
+        end
+      end
     end
   end
 end

data/lib/connectors_sdk/base/extractor.rb

@@ -188,16 +188,19 @@ module ConnectorsSdk
       end
 
       def permissions(source_user_id, &block)
+        result = []
         Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.permissions") do
           with_auth_tokens_and_retry do
             Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.yield_permissions") do
               yield_permissions(source_user_id) do |permissions|
                 log_info("Extracted #{permissions.size} permissions for source user #{source_user_id}")
+                result = permissions
                 block.call(permissions) if block_given?
               end
             end
           end
         end
+        result.each
       end
 
       ConnectorsShared::Logger::SUPPORTED_LOG_LEVELS.each do |log_level|

data/lib/connectors_sdk/base/registry.rb

@@ -24,11 +24,14 @@ module ConnectorsSdk
 
     REGISTRY = Factory.new
 
+    require_relative '../stub_connector/connector'
+    REGISTRY.register(ConnectorsSdk::StubConnector::Connector::SERVICE_TYPE, ConnectorsSdk::StubConnector::Connector)
+
     # loading plugins (might replace this with a directory scan and conventions on names)
-    require_relative '../share_point/http_call_wrapper'
-    require_relative '../confluence_cloud//http_call_wrapper'
+    require_relative '../share_point/connector'
+    require_relative '../confluence_cloud/connector'
 
-    REGISTRY.register(ConnectorsSdk::SharePoint::HttpCallWrapper::SERVICE_TYPE, ConnectorsSdk::SharePoint::HttpCallWrapper)
-    REGISTRY.register(ConnectorsSdk::ConfluenceCloud::HttpCallWrapper::SERVICE_TYPE, ConnectorsSdk::ConfluenceCloud::HttpCallWrapper)
+    REGISTRY.register(ConnectorsSdk::SharePoint::Connector::SERVICE_TYPE, ConnectorsSdk::SharePoint::Connector)
+    REGISTRY.register(ConnectorsSdk::ConfluenceCloud::Connector::SERVICE_TYPE, ConnectorsSdk::ConfluenceCloud::Connector)
   end
 end

data/lib/connectors_sdk/confluence/extractor.rb

@@ -82,7 +82,7 @@ module ConnectorsSdk
       private
 
       def content_base_url
-        'https://workplace-search.atlassian.net/wiki'
+        config.base_url
       end
 
       def yield_spaces

data/lib/connectors_sdk/confluence_cloud/{http_call_wrapper.rb → connector.rb}

@@ -10,19 +10,44 @@ require 'connectors_sdk/atlassian/config'
 require 'connectors_sdk/confluence_cloud/extractor'
 require 'connectors_sdk/confluence_cloud/authorization'
 require 'connectors_sdk/confluence_cloud/custom_client'
-require 'connectors_sdk/base/http_call_wrapper'
+require 'connectors_sdk/base/connector'
 
 module ConnectorsSdk
   module ConfluenceCloud
-    class HttpCallWrapper < ConnectorsSdk::Base::HttpCallWrapper
+    class Connector < ConnectorsSdk::Base::Connector
       SERVICE_TYPE = 'confluence_cloud'
 
-      def name
+      def compare_secrets(params)
+        missing_secrets?(params)
+
+        {
+          :equivalent => params[:secret] == params[:other_secret]
+        }
+      end
+
+      def display_name
         'Confluence Cloud'
       end
 
-      def service_type
-        SERVICE_TYPE
+      def connection_requires_redirect
+        true
+      end
+
+      def configurable_fields
+        [
+          {
+            'key' => 'base_url',
+            'label' => 'Base URL'
+          },
+          {
+            'key' => 'client_id',
+            'label' => 'Client ID'
+          },
+          {
+            'key' => 'client_secret',
+            'label' => 'Client Secret'
+          },
+        ]
       end
 
       private
@@ -44,7 +69,7 @@ module ConnectorsSdk
       end
 
       def config(params)
-        ConnectorsSdk::Atlassian::Config.new(:base_url => base_url(params[:cloud_id]), :cursors => params.fetch(:cursors, {}) || {})
+        ConnectorsSdk::Atlassian::Config.new(:base_url => "#{params[:external_connector_base_url]}/wiki", :cursors => params.fetch(:cursors, {}) || {})
       end
 
       def health_check(params)

data/lib/connectors_sdk/share_point/{http_call_wrapper.rb → connector.rb}

@@ -6,22 +6,46 @@
 
 # frozen_string_literal: true
 
+require 'connectors_sdk/base/connector'
 require 'connectors_sdk/office365/config'
 require 'connectors_sdk/share_point/extractor'
 require 'connectors_sdk/share_point/authorization'
-require 'connectors_sdk/base/http_call_wrapper'
 
 module ConnectorsSdk
   module SharePoint
-    class HttpCallWrapper < ConnectorsSdk::Base::HttpCallWrapper
+    class Connector < ConnectorsSdk::Base::Connector
       SERVICE_TYPE = 'share_point'
 
-      def name
-        'SharePoint'
+      def compare_secrets(params)
+        missing_secrets?(params)
+
+        previous_user = client(:access_token => params[:other_secret][:access_token]).me
+        equivalent = previous_user.nil? ? false : previous_user.id == client(:access_token => params[:secret][:access_token]).me&.id
+
+        {
+          :equivalent => equivalent
+        }
+      end
+
+      def display_name
+        'SharePoint Online'
+      end
+
+      def connection_requires_redirect
+        true
       end
 
-      def service_type
-        SERVICE_TYPE
+      def configurable_fields
+        [
+          {
+            'key' => 'client_id',
+            'label' => 'Client ID'
+          },
+          {
+            'key' => 'client_secret',
+            'label' => 'Client Secret'
+          },
+        ]
       end
 
       private

data/lib/connectors_sdk/stub_connector/connector.rb

@@ -0,0 +1,62 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/connector'
+
+module ConnectorsSdk
+  module StubConnector
+    class Connector < ConnectorsSdk::Base::Connector
+      SERVICE_TYPE = 'stub_connector'
+
+      def display_name
+        'Stub Connector'
+      end
+
+      def configurable_fields
+        [
+          {
+            'key' => 'third_party_url',
+            'label' => 'Third Party URL'
+          },
+          {
+            'key' => 'third_party_api_key',
+            'label' => 'Third Party API Key'
+          }
+        ]
+      end
+
+      def health_check(_params)
+        true
+      end
+
+      def document_batch(_params)
+        results = 30.times.map do |i|
+          {
+            :action => :create_or_update,
+            :document => {
+              :id => "document_#{i}",
+              :type => 'document',
+              :body => "contents for document number: #{i}"
+            },
+            :download => nil
+          }
+        end
+
+        [results, {}, true]
+      end
+
+      def deleted(_params)
+        []
+      end
+
+      def permissions(_params)
+        []
+      end
+    end
+  end
+end

data/lib/connectors_shared/constants.rb

@@ -10,5 +10,17 @@ module ConnectorsShared
     SUBEXTRACTOR_RESERVED_FIELDS = %w[_subextracted_as_of _subextracted_version].freeze
     ALLOW_FIELD = '_allow_permissions'.freeze
     DENY_FIELD = '_deny_permissions'.freeze
+
+    # The following section reads as following:
+    # The job will extract documents until the job queue size will reach
+    # JOB_QUEUE_SIZE_IDLE_THRESHOLD items. After that, the job will attempt to sleep
+    # for IDLE_SLEEP_TIME seconds and check the queue size again. If the queue is still
+    # full, it will sleep for maximum MAX_IDDLE_ATTEMPTS times, and if the queue is still
+    # full, then job will be terminated.
+    JOB_QUEUE_SIZE_IDLE_THRESHOLD = 500 # How many documents the job queue stores until it sleeps
+    IDLE_SLEEP_TIME = 10 # For how long job queue will sleep before checking the queue size again
+    MAX_IDLE_ATTEMPTS = 30 # How many consecutive times job will try to sleep until it's destroyed
+
+    STALE_JOB_TIMEOUT = 60 * 30 # Time in seconds after which the job will be cleaned up if the job is considered stuck
   end
 end

data/lib/connectors_shared/middleware/restrict_hostnames.rb

@@ -36,7 +36,7 @@ module ConnectorsShared
       def ips_from_hosts(hosts)
         hosts&.flat_map do |host|
           if URL_PATTERN.match(host)
-            lookup_ips(URI.parse(host).host)
+            lookup_ips(Addressable::URI.parse(host).hostname)
           elsif Resolv::IPv4::Regex.match(host) || Resolv::IPv6::Regex.match(host)
             IPAddr.new(host)
           else

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_sdk
 version: !ruby/object:Gem::Version
-  version: 8.3.0.0.pre.20220517T144653Z
+  version: 8.3.0.0
 platform: ruby
 authors:
 - Elastic
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-17 00:00:00.000000000 Z
+date: 2022-05-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -66,17 +66,17 @@ files:
 - lib/connectors_sdk/base/adapter.rb
 - lib/connectors_sdk/base/authorization.rb
 - lib/connectors_sdk/base/config.rb
+- lib/connectors_sdk/base/connector.rb
 - lib/connectors_sdk/base/custom_client.rb
 - lib/connectors_sdk/base/extractor.rb
-- lib/connectors_sdk/base/http_call_wrapper.rb
 - lib/connectors_sdk/base/registry.rb
 - lib/connectors_sdk/confluence/adapter.rb
 - lib/connectors_sdk/confluence/custom_client.rb
 - lib/connectors_sdk/confluence/extractor.rb
 - lib/connectors_sdk/confluence_cloud/authorization.rb
+- lib/connectors_sdk/confluence_cloud/connector.rb
 - lib/connectors_sdk/confluence_cloud/custom_client.rb
 - lib/connectors_sdk/confluence_cloud/extractor.rb
-- lib/connectors_sdk/confluence_cloud/http_call_wrapper.rb
 - lib/connectors_sdk/helpers/atlassian_time_formatter.rb
 - lib/connectors_sdk/office365/adapter.rb
 - lib/connectors_sdk/office365/config.rb
@@ -84,8 +84,9 @@ files:
 - lib/connectors_sdk/office365/extractor.rb
 - lib/connectors_sdk/share_point/adapter.rb
 - lib/connectors_sdk/share_point/authorization.rb
+- lib/connectors_sdk/share_point/connector.rb
 - lib/connectors_sdk/share_point/extractor.rb
-- lib/connectors_sdk/share_point/http_call_wrapper.rb
+- lib/connectors_sdk/stub_connector/connector.rb
 - lib/connectors_shared.rb
 - lib/connectors_shared/constants.rb
 - lib/connectors_shared/errors.rb
@@ -103,9 +104,9 @@ homepage: https://github.com/elastic/connectors
 licenses:
 - Elastic-2.0
 metadata:
-  revision: 9f25f35e17ffb36dfda754d657794ed9b5d2d75a
+  revision: 8fe65b05c8d8c89d62d73e7d717238a5393d42f5
   repository: git@github.com:elastic/connectors.git
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -116,12 +117,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.0.3.1
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Gem containing apis used by Enterprise Search and implementations of Connectors
 test_files: []