connectors_sdk 8.3.0.0.pre.20220414T060419Z → 8.3.0.0

data/lib/connectors_sdk/confluence_cloud/connector.rb

@@ -0,0 +1,84 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/atlassian/config'
+require 'connectors_sdk/confluence_cloud/extractor'
+require 'connectors_sdk/confluence_cloud/authorization'
+require 'connectors_sdk/confluence_cloud/custom_client'
+require 'connectors_sdk/base/connector'
+
+module ConnectorsSdk
+  module ConfluenceCloud
+    class Connector < ConnectorsSdk::Base::Connector
+      SERVICE_TYPE = 'confluence_cloud'
+
+      def compare_secrets(params)
+        missing_secrets?(params)
+
+        {
+          :equivalent => params[:secret] == params[:other_secret]
+        }
+      end
+
+      def display_name
+        'Confluence Cloud'
+      end
+
+      def connection_requires_redirect
+        true
+      end
+
+      def configurable_fields
+        [
+          {
+            'key' => 'base_url',
+            'label' => 'Base URL'
+          },
+          {
+            'key' => 'client_id',
+            'label' => 'Client ID'
+          },
+          {
+            'key' => 'client_secret',
+            'label' => 'Client Secret'
+          },
+        ]
+      end
+
+      private
+
+      def extractor_class
+        ConnectorsSdk::ConfluenceCloud::Extractor
+      end
+
+      def authorization
+        ConnectorsSdk::ConfluenceCloud::Authorization
+      end
+
+      def client(params)
+        ConnectorsSdk::ConfluenceCloud::CustomClient.new(:base_url => base_url(params[:cloud_id]), :access_token => params[:access_token])
+      end
+
+      def custom_client_error
+        ConnectorsSdk::Atlassian::CustomClient::ClientError
+      end
+
+      def config(params)
+        ConnectorsSdk::Atlassian::Config.new(:base_url => "#{params[:external_connector_base_url]}/wiki", :cursors => params.fetch(:cursors, {}) || {})
+      end
+
+      def health_check(params)
+        client(params).me
+      end
+
+      def base_url(cloud_id)
+        "https://api.atlassian.com/ex/confluence/#{cloud_id}"
+      end
+    end
+  end
+end

data/lib/connectors_sdk/confluence_cloud/custom_client.rb

@@ -0,0 +1,61 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/confluence/custom_client'
+
+module ConnectorsSdk
+  module ConfluenceCloud
+    class CustomClient < ConnectorsSdk::Confluence::CustomClient
+      def user_groups(account_id, limit: 200, start: 0)
+        size = Float::INFINITY
+
+        groups = []
+
+        while start + limit < size
+          params = {
+            :start => start,
+            :limit => limit,
+            :accountId => account_id
+          }
+          response = get('rest/api/user/memberof', params)
+          result = Hashie::Mash.new(parse_and_raise_if_necessary!(response))
+          size = result.size
+          start += limit
+          groups.concat(result.results)
+        end
+
+        groups
+      end
+
+      def user(account_id, expand: [])
+        params = {
+          :accountId => account_id
+        }
+        if expand.present?
+          params[:expand] = case expand
+                            when Array
+                              expand.join(',')
+                            when String
+                              expand
+                            else
+                              expand.to_s
+                            end
+        end
+        response = get('rest/api/user', params)
+        Hashie::Mash.new(parse_and_raise_if_necessary!(response))
+      rescue ConnectorsSdk::Atlassian::CustomClient::ClientError => e
+        if e.status_code == 404
+          ConnectorsShared::Logger.warn("Could not find a user with account id #{account_id}")
+          nil
+        else
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/connectors_sdk/confluence_cloud/extractor.rb

@@ -0,0 +1,59 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/confluence/extractor'
+
+module ConnectorsSdk
+  module ConfluenceCloud
+    class Extractor < ConnectorsSdk::Confluence::Extractor
+
+      def yield_permissions(source_user_id)
+        # yield empty permissions if the user is suspended or deleted
+        user = client.user(source_user_id, :expand => 'operations')
+        if user.nil? || user.operations.blank?
+          yield [] and return
+        end
+
+        # refresh space permissions if not initialized
+        if @space_permissions_cache.nil?
+          @space_permissions_cache = {}
+          yield_spaces do |space|
+            if config.index_permissions
+              get_space_permissions(space)
+            end
+          end
+        end
+
+        direct_spaces = get_user_spaces(source_user_id)
+        indirect_spaces = []
+
+        group_permissions = []
+        client.user_groups(source_user_id).each do |group|
+          group_name = group.name
+          group_spaces = get_group_spaces(group_name)
+          indirect_spaces << group_spaces
+          group_permissions << "group:#{group_name}"
+        end
+
+        total_user_spaces = indirect_spaces.flatten.concat(direct_spaces).uniq
+        user_permissions = ["user:#{source_user_id}"]
+          .concat(group_permissions)
+          .product(total_user_spaces)
+          .collect { |permission, space| "#{space}/#{permission}" }
+
+        yield user_permissions.flatten.uniq.sort
+      end
+
+      def download(item)
+        content = item[:content]
+        parent_id = content.dig('container', 'id')
+        client.download("#{client.base_url}/wiki/rest/api/content/#{parent_id}/child/attachment/#{content['id']}/download").body
+      end
+    end
+  end
+end

data/lib/connectors_sdk/helpers/atlassian_time_formatter.rb

@@ -0,0 +1,10 @@
+module ConnectorsSdk
+  module Helpers
+    module AtlassianTimeFormatter
+      def format_time(time)
+        time = Time.parse(time) if time.is_a?(String)
+        time.strftime('%Y-%m-%d %H:%M')
+      end
+    end
+  end
+end

data/lib/connectors_sdk/office365/adapter.rb

@@ -11,11 +11,11 @@ require 'connectors_sdk/base/adapter'
 module ConnectorsSdk
   module Office365
     class Adapter < ConnectorsSdk::Base::Adapter
-      def self.swiftype_document_from_file(_file)
+      def self.es_document_from_file(_file)
         raise NotImplementedError
       end
 
-      def self.swiftype_document_from_folder(_folder)
+      def self.es_document_from_folder(_folder)
         raise NotImplementedError
       end
 
@@ -26,7 +26,7 @@ module ConnectorsSdk
           @item = item
         end
 
-        def self.convert_id_to_fp_id(_id)
+        def self.convert_id_to_es_id(_id)
           raise NotImplementedError
         end
 
@@ -41,10 +41,10 @@ module ConnectorsSdk
           ConnectorsSdk::Office365::Adapter.normalize_path("#{parent_folder_path}/#{item.name}")
         end
 
-        def to_swiftype_document
+        def to_es_document
           {
             :_fields_to_preserve => ConnectorsSdk::Office365::Adapter.fields_to_preserve,
-            :id => self.class.convert_id_to_fp_id(item.id),
+            :id => self.class.convert_id_to_es_id(item.id),
             :path => get_path(item),
             :title => item.name,
             :url => item.webUrl,
@@ -96,7 +96,7 @@ module ConnectorsSdk
       end
 
       class FileGraphItem < GraphItem
-        def self.convert_id_to_fp_id(_id)
+        def self.convert_id_to_es_id(_id)
           raise NotImplementedError
         end
 
@@ -132,7 +132,7 @@ module ConnectorsSdk
       end
 
       class PackageGraphItem < GraphItem
-        def self.convert_id_to_fp_id(id)
+        def self.convert_id_to_es_id(id)
           raise NotImplementedError
         end
 

data/lib/connectors_sdk/office365/config.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require 'connectors_sdk/base/config'
+require 'connectors_sdk/office365/extractor'
 
 module ConnectorsSdk
   module Office365

data/lib/connectors_sdk/office365/custom_client.rb

@@ -6,9 +6,11 @@
 
 # frozen_string_literal: true
 
+require 'hashie/mash'
+
 require 'connectors_sdk/base/custom_client'
+require 'connectors_sdk/office365/extractor'
 require 'connectors_shared'
-require 'hashie/mash'
 
 module ConnectorsSdk
   module Office365
@@ -50,7 +52,6 @@ module ConnectorsSdk
       def initialize(access_token:, cursors: {}, ensure_fresh_auth: nil)
         @access_token = access_token
         @cursors = cursors || {}
-        @cursors[ConnectorsSdk::Office365::Extractor::DRIVE_IDS_CURSOR_KEY] ||= {}
         super(:ensure_fresh_auth => ensure_fresh_auth)
       end
 
@@ -75,12 +76,11 @@ module ConnectorsSdk
         # recently created groups (new Private Team site will be there) to reduce friction and index this site
         # earlier.
         # See: https://github.com/elastic/ent-search/pull/3581
-        share_point_sites = (sites(:fields => %w[id]) + recent_share_point_group_sites(:fields => %[id]))
+        share_point_sites = (sites(:fields => %w[id,name]) + recent_share_point_group_sites(:fields => %w[id,name]))
 
         share_point_sites
-          .map(&:id)
-          .uniq
-          .map { |site_id| site_drives(site_id, :fields => fields) }
+          .uniq(&:id)
+          .map { |site| site_drives(site, :fields => fields) }
           .flatten
           .compact
       end
@@ -102,27 +102,23 @@ module ConnectorsSdk
         request_all(:endpoint => 'sites/', :fields => fields, :additional_query_params => { :search => '', :top => 10 })
       end
 
-      def site_drives(site_id, fields: [])
+      def site_drives(site, fields: [])
         document_libraries(
-          request_all(:endpoint => "sites/#{site_id}/drives/", :fields => fields)
-        )
+          request_all(:endpoint => "sites/#{site.id}/drives/", :fields => fields)
+        ).map do |drive|
+          drive.site_name = site.name
+          drive
+        end
       rescue ClientError => e
         ConnectorsShared::Logger.info("Received response of #{e.status_code} trying to get drive for Site with Id = #{site_id}: #{e.message}")
         nil
       end
 
-      def list_items(drive_id, fields: [], break_after_page: false)
+      def list_items(drive_id, fields: [])
         # MSFT Graph API does not have a recursive list items, have to do this dfs style
-
-        stack = if break_after_page && cursors['page_cursor'].present?
-                  cursors.delete('page_cursor')
-                else
-                  [get_root_item(drive_id, ['id']).id]
-                end
-
+        stack = [get_root_item(drive_id, ['id']).id]
         # We rely on the id field below to perform our DFS
         fields_with_id = fields.any? ? fields | ['id'] : fields
-        yielded = 0
         while stack.any?
           folder_id = stack.pop
           item_children(drive_id, folder_id, :fields => fields_with_id) do |item|
@@ -130,14 +126,8 @@ module ConnectorsSdk
               stack << item.id
             end
             yield item
-
-            yielded += 1
           end
 
-          if break_after_page && yielded >= 100 && stack.any?
-            cursors['page_cursor'] = stack.dup
-            break
-          end
         end
       end
 
@@ -145,19 +135,16 @@ module ConnectorsSdk
         request_endpoint(:endpoint => "drives/#{drive_id}/items/#{item_id}/permissions").value
       end
 
-      def list_changes(drive_id:, start_delta_link: nil, last_modified: nil, break_after_page: false)
+      def list_changes(drive_id:, start_delta_link: nil, last_modified: nil)
         query_params = { :'$select' => %w(id content.downloadUrl lastModifiedDateTime lastModifiedBy root deleted file folder package name webUrl createdBy createdDateTime size).join(',') }
         response =
-          if break_after_page && cursors['page_cursor'].present?
-            request_json(:url => cursors.delete('page_cursor'))
-          elsif start_delta_link.nil?
+          if start_delta_link.nil?
             endpoint = "drives/#{drive_id}/root/delta"
             request_endpoint(:endpoint => endpoint, :query_params => query_params)
           else
             request_json(:url => start_delta_link, :query_params => query_params)
           end
 
-        yielded = 0
         loop do
           response.value.each do |change|
             # MSFT Graph API does not allow us to view "changes" in chronological order, so if there is no cursor,
@@ -165,25 +152,18 @@ module ConnectorsSdk
             # since to get another cursor, we would have to go through all the changes anyway
             next if last_modified.present? && Time.parse(change.lastModifiedDateTime) < last_modified
             next if change.root # We don't want to index the root of the drive
-
             yield change
-            yielded += 1
-          end
-
-          if break_after_page && yielded >= 100 && response['@odata.nextLink'].present?
-            cursors['page_cursor'] = response['@odata.nextLink']
-            break
           end
 
           break if response['@odata.nextLink'].nil?
           response = request_json(:url => response['@odata.nextLink'])
         end
 
-        cursors[ConnectorsSdk::Office365::Extractor::DRIVE_IDS_CURSOR_KEY][drive_id] = response['@odata.deltaLink']
+        cursors[drive_id] = response['@odata.deltaLink']
       end
 
       def get_latest_delta_link(drive_id)
-        cursors[ConnectorsSdk::Office365::Extractor::DRIVE_IDS_CURSOR_KEY][drive_id] || exhaustively_get_delta_link(drive_id)
+        cursors[drive_id] || exhaustively_get_delta_link(drive_id)
       end
 
       def exhaustively_get_delta_link(drive_id)
@@ -203,6 +183,7 @@ module ConnectorsSdk
       def download_item(download_url)
         request(:url => download_url) do |request|
           request.options.params_encoder = Office365DownloadParamsEncoder
+          request.options.timeout = 30
         end.body
       end
 
@@ -228,7 +209,7 @@ module ConnectorsSdk
 
         groups(:fields => %w(id createdDateTime))
           .select { |group| group.createdDateTime > created_date_time_threshold }
-          .map { |group| group_root_site(group.id, :fields => %w[id]) }.compact
+          .map { |group| group_root_site(group.id, :fields => fields) }.compact
       end
 
       def document_libraries(drives)

data/lib/connectors_sdk/office365/extractor.rb

@@ -14,52 +14,34 @@ module ConnectorsSdk
     class Extractor < ConnectorsSdk::Base::Extractor
       DRIVE_IDS_CURSOR_KEY = 'drive_ids'.freeze
 
-      def yield_document_changes(modified_since: nil, break_after_page: false, &block)
+      def yield_document_changes(modified_since: nil, &block)
         drives_to_index.each do |drive|
           drive_id = drive.id
-
-          if break_after_page
-            current_drive_id = config.cursors['current_drive_id']
-            if current_drive_id.present? && current_drive_id > drive_id # they come alpha sorted
-              next
-            end
-            config.cursors['current_drive_id'] = drive_id
-          end
-
           drive_owner_name = drive.dig(:owner, :user, :displayName)
           drive_name = drive.name
+          site_name = drive.site_name
 
           drive_id_to_delta_link = config.cursors.fetch(DRIVE_IDS_CURSOR_KEY, {})
           begin
             if start_delta_link = drive_id_to_delta_link[drive_id]
               log_debug("Starting an incremental crawl with cursor for #{service_type.classify} with drive_id: #{drive_id}")
               begin
-                yield_changes(drive_id, :start_delta_link => start_delta_link, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+                yield_changes(drive_id, :start_delta_link => start_delta_link, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
               rescue ConnectorsSdk::Office365::CustomClient::Office365InvalidCursorsError
                 log_warn("Error listing changes with start_delta_link: #{start_delta_link}, falling back to full crawl")
-                yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+                yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
               end
             elsif modified_since.present?
               log_debug("Starting an incremental crawl using last_modified (no cursor found) for #{service_type.classify} with drive_id: #{drive_id}")
-              yield_changes(drive_id, :last_modified => modified_since, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+              yield_changes(drive_id, :last_modified => modified_since, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
             else
               log_debug("Starting a full crawl #{service_type.classify} with drive_id: #{drive_id}")
-              yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+              yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
             end
           rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
             log_warn("Error searching and listing drive #{drive_id}")
             capture_exception(e)
           end
-
-          if break_after_page && config.cursors['page_cursor'].present?
-            break
-          end
-        end
-
-        if break_after_page && config.cursors['page_cursor'].blank?
-          @completed = true
-          config.overwrite_cursors!(retrieve_latest_cursors)
-          log_debug("Completed #{modified_since.nil? ? 'full' : 'incremental'} extraction")
         end
 
         nil
@@ -132,7 +114,7 @@ module ConnectorsSdk
         @existing_drive_item_ids ||= Set.new.tap do |ids|
           drives_to_index.each do |drive|
             client.list_items(drive.id) do |item|
-              ids << convert_id_to_fp_id(item.id)
+              ids << convert_id_to_es_id(item.id)
             end
           end
         end
@@ -142,7 +124,7 @@ module ConnectorsSdk
         raise NotImplementedError
       end
 
-      def convert_id_to_fp_id(_id)
+      def convert_id_to_es_id(_id)
         raise NotImplementedError
       end
 
@@ -156,11 +138,12 @@ module ConnectorsSdk
         ConnectorsShared::ExceptionTracking.capture_exception(office365_client_error, options)
       end
 
-      def yield_drive_items(drive_id, drive_owner_name:, drive_name:, break_after_page: false, &block)
-        client.list_items(drive_id, break_after_page: break_after_page) do |item|
+      def yield_drive_items(drive_id, drive_owner_name:, drive_name:, site_name:, &block)
+        client.list_items(drive_id) do |item|
           yield_single_document_change(:identifier => "Office365 change: #{item&.id} (#{Office365::Adapter::GraphItem.get_path(item)})") do
             item.drive_owner_name = drive_owner_name
             item.drive_name = drive_name
+            item.site_name = site_name
             yield_create_or_update(drive_id, item, &block)
           end
         end
@@ -170,15 +153,16 @@ module ConnectorsSdk
         if item.deleted.nil?
           yield_create_or_update(drive_id, item, &block)
         else
-          yield :delete, convert_id_to_fp_id(item.id)
+          yield :delete, convert_id_to_es_id(item.id)
         end
       end
 
-      def yield_changes(drive_id, drive_owner_name:, drive_name:, start_delta_link: nil, last_modified: nil, break_after_page: false, &block)
-        client.list_changes(:drive_id => drive_id, :start_delta_link => start_delta_link, :last_modified => last_modified, :break_after_page => break_after_page) do |item|
+      def yield_changes(drive_id, drive_owner_name:, drive_name:, site_name:, start_delta_link: nil, last_modified: nil, &block)
+        client.list_changes(:drive_id => drive_id, :start_delta_link => start_delta_link, :last_modified => last_modified) do |item|
           yield_single_document_change(:identifier => "Office365 change: #{item&.id} (#{Office365::Adapter::GraphItem.get_path(item)})") do
             item.drive_owner_name = drive_owner_name
             item.drive_name = drive_name
+            item.site_name = site_name
             yield_correct_actions_and_converted_item(drive_id, item, &block)
           end
         end
@@ -210,11 +194,11 @@ module ConnectorsSdk
 
       def generate_document(item)
         if item.file
-          adapter.swiftype_document_from_file(item)
+          adapter.es_document_from_file(item)
         elsif item.folder
-          adapter.swiftype_document_from_folder(item)
+          adapter.es_document_from_folder(item)
         elsif item.package
-          adapter.swiftype_document_from_package(item)
+          adapter.es_document_from_package(item)
         else
           raise "Unexpected Office 365 item type for item #{item}"
         end

data/lib/connectors_sdk/share_point/adapter.rb

@@ -10,36 +10,48 @@ require 'connectors_sdk/office365/adapter'
 
 module ConnectorsSdk
   module SharePoint
+    module SitePrefix
+      def get_path(item)
+        item.site_name.present? ? "/sites/#{item.site_name}#{super}" : super
+      end
+    end
+
     class Adapter < Office365::Adapter
       generate_id_helpers :share_point, 'share_point'
 
-      def self.swiftype_document_from_file(file)
-        FileGraphItem.new(file).to_swiftype_document
+      def self.es_document_from_file(file)
+        FileGraphItem.new(file).to_es_document
       end
 
-      def self.swiftype_document_from_folder(folder)
-        FolderGraphItem.new(folder).to_swiftype_document
+      def self.es_document_from_folder(folder)
+        FolderGraphItem.new(folder).to_es_document
       end
 
-      def self.swiftype_document_from_package(package)
-        PackageGraphItem.new(package).to_swiftype_document
+      def self.es_document_from_package(package)
+        PackageGraphItem.new(package).to_es_document
       end
 
       class FileGraphItem < Office365::Adapter::FileGraphItem
-        def self.convert_id_to_fp_id(id)
-          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+        include SitePrefix
+
+        def self.convert_id_to_es_id(id)
+          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end
       end
 
       class FolderGraphItem < Office365::Adapter::FolderGraphItem
-        def self.convert_id_to_fp_id(id)
-          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+        include SitePrefix
+
+        def self.convert_id_to_es_id(id)
+          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end
       end
 
       class PackageGraphItem < Office365::Adapter::PackageGraphItem
-        def self.convert_id_to_fp_id(id)
-          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+        include SitePrefix
+
+        def self.convert_id_to_es_id(id)
+          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end
       end
     end