connectors_sdk 8.2.0.0 → 8.3.0.0.pre.20220517T144653Z

data/lib/connectors_sdk/office365/adapter.rb

@@ -11,11 +11,11 @@ require 'connectors_sdk/base/adapter'
 module ConnectorsSdk
   module Office365
     class Adapter < ConnectorsSdk::Base::Adapter
-      def self.swiftype_document_from_file(_file)
+      def self.es_document_from_file(_file)
         raise NotImplementedError
       end
 
-      def self.swiftype_document_from_folder(_folder)
+      def self.es_document_from_folder(_folder)
         raise NotImplementedError
       end
 
@@ -26,7 +26,7 @@ module ConnectorsSdk
           @item = item
         end
 
-        def self.convert_id_to_fp_id(_id)
+        def self.convert_id_to_es_id(_id)
           raise NotImplementedError
         end
 
@@ -41,10 +41,10 @@ module ConnectorsSdk
           ConnectorsSdk::Office365::Adapter.normalize_path("#{parent_folder_path}/#{item.name}")
         end
 
-        def to_swiftype_document
+        def to_es_document
           {
             :_fields_to_preserve => ConnectorsSdk::Office365::Adapter.fields_to_preserve,
-            :id => self.class.convert_id_to_fp_id(item.id),
+            :id => self.class.convert_id_to_es_id(item.id),
             :path => get_path(item),
             :title => item.name,
             :url => item.webUrl,
@@ -96,7 +96,7 @@ module ConnectorsSdk
       end
 
       class FileGraphItem < GraphItem
-        def self.convert_id_to_fp_id(_id)
+        def self.convert_id_to_es_id(_id)
           raise NotImplementedError
         end
 
@@ -132,7 +132,7 @@ module ConnectorsSdk
       end
 
       class PackageGraphItem < GraphItem
-        def self.convert_id_to_fp_id(id)
+        def self.convert_id_to_es_id(id)
           raise NotImplementedError
         end
 

data/lib/connectors_sdk/office365/config.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require 'connectors_sdk/base/config'
+require 'connectors_sdk/office365/extractor'
 
 module ConnectorsSdk
   module Office365

data/lib/connectors_sdk/office365/custom_client.rb

@@ -6,9 +6,11 @@
 
 # frozen_string_literal: true
 
+require 'hashie/mash'
+
 require 'connectors_sdk/base/custom_client'
+require 'connectors_sdk/office365/extractor'
 require 'connectors_shared'
-require 'hashie/mash'
 
 module ConnectorsSdk
   module Office365
@@ -50,7 +52,6 @@ module ConnectorsSdk
       def initialize(access_token:, cursors: {}, ensure_fresh_auth: nil)
         @access_token = access_token
         @cursors = cursors || {}
-        @cursors[ConnectorsSdk::Office365::Extractor::DRIVE_IDS_CURSOR_KEY] ||= {}
         super(:ensure_fresh_auth => ensure_fresh_auth)
       end
 
@@ -75,12 +76,11 @@ module ConnectorsSdk
         # recently created groups (new Private Team site will be there) to reduce friction and index this site
         # earlier.
         # See: https://github.com/elastic/ent-search/pull/3581
-        share_point_sites = (sites(:fields => %w[id]) + recent_share_point_group_sites(:fields => %[id]))
+        share_point_sites = (sites(:fields => %w[id,name]) + recent_share_point_group_sites(:fields => %w[id,name]))
 
         share_point_sites
-          .map(&:id)
-          .uniq
-          .map { |site_id| site_drives(site_id, :fields => fields) }
+          .uniq(&:id)
+          .map { |site| site_drives(site, :fields => fields) }
           .flatten
           .compact
       end
@@ -102,47 +102,32 @@ module ConnectorsSdk
         request_all(:endpoint => 'sites/', :fields => fields, :additional_query_params => { :search => '', :top => 10 })
       end
 
-      def site_drives(site_id, fields: [])
+      def site_drives(site, fields: [])
         document_libraries(
-          request_all(:endpoint => "sites/#{site_id}/drives/", :fields => fields)
-        )
+          request_all(:endpoint => "sites/#{site.id}/drives/", :fields => fields)
+        ).map do |drive|
+          drive.site_name = site.name
+          drive
+        end
       rescue ClientError => e
         ConnectorsShared::Logger.info("Received response of #{e.status_code} trying to get drive for Site with Id = #{site_id}: #{e.message}")
         nil
       end
 
-      def list_items(drive_id, fields: [], break_after_page: false)
+      def list_items(drive_id, fields: [])
         # MSFT Graph API does not have a recursive list items, have to do this dfs style
-
-        stack = if break_after_page && cursors['page_cursor'].present?
-                  cursors.delete('page_cursor')
-                else
-                  [get_root_item(drive_id, ['id']).id]
-                end
-
+        stack = [get_root_item(drive_id, ['id']).id]
         # We rely on the id field below to perform our DFS
         fields_with_id = fields.any? ? fields | ['id'] : fields
-        yielded = 0
         while stack.any?
           folder_id = stack.pop
-          item_children(drive_id, folder_id, :fields => fields_with_id, :break_after_page => break_after_page) do |item|
+          item_children(drive_id, folder_id, :fields => fields_with_id) do |item|
             if item.folder
               stack << item.id
             end
             yield item
-
-            yielded += 1
           end
 
-          if break_after_page && yielded >= 100
-            if cursors['item_children_next_link'].present?
-              stack << folder_id
-            end
-            if stack.any?
-              cursors['page_cursor'] = stack.dup
-              break
-            end
-          end
         end
       end
 
@@ -150,19 +135,16 @@ module ConnectorsSdk
         request_endpoint(:endpoint => "drives/#{drive_id}/items/#{item_id}/permissions").value
       end
 
-      def list_changes(drive_id:, start_delta_link: nil, last_modified: nil, break_after_page: false)
+      def list_changes(drive_id:, start_delta_link: nil, last_modified: nil)
         query_params = { :'$select' => %w(id content.downloadUrl lastModifiedDateTime lastModifiedBy root deleted file folder package name webUrl createdBy createdDateTime size).join(',') }
         response =
-          if break_after_page && cursors['page_cursor'].present?
-            request_json(:url => cursors.delete('page_cursor'))
-          elsif start_delta_link.nil?
+          if start_delta_link.nil?
             endpoint = "drives/#{drive_id}/root/delta"
             request_endpoint(:endpoint => endpoint, :query_params => query_params)
           else
             request_json(:url => start_delta_link, :query_params => query_params)
           end
 
-        yielded = 0
         loop do
           response.value.each do |change|
             # MSFT Graph API does not allow us to view "changes" in chronological order, so if there is no cursor,
@@ -170,25 +152,18 @@ module ConnectorsSdk
             # since to get another cursor, we would have to go through all the changes anyway
             next if last_modified.present? && Time.parse(change.lastModifiedDateTime) < last_modified
             next if change.root # We don't want to index the root of the drive
-
             yield change
-            yielded += 1
-          end
-
-          if break_after_page && yielded >= 100 && response['@odata.nextLink'].present?
-            cursors['page_cursor'] = response['@odata.nextLink']
-            break
           end
 
           break if response['@odata.nextLink'].nil?
           response = request_json(:url => response['@odata.nextLink'])
         end
 
-        cursors[ConnectorsSdk::Office365::Extractor::DRIVE_IDS_CURSOR_KEY][drive_id] = response['@odata.deltaLink']
+        cursors[drive_id] = response['@odata.deltaLink']
       end
 
       def get_latest_delta_link(drive_id)
-        cursors[ConnectorsSdk::Office365::Extractor::DRIVE_IDS_CURSOR_KEY][drive_id] || exhaustively_get_delta_link(drive_id)
+        cursors[drive_id] || exhaustively_get_delta_link(drive_id)
       end
 
       def exhaustively_get_delta_link(drive_id)
@@ -208,6 +183,7 @@ module ConnectorsSdk
       def download_item(download_url)
         request(:url => download_url) do |request|
           request.options.params_encoder = Office365DownloadParamsEncoder
+          request.options.timeout = 30
         end.body
       end
 
@@ -233,7 +209,7 @@ module ConnectorsSdk
 
         groups(:fields => %w(id createdDateTime))
           .select { |group| group.createdDateTime > created_date_time_threshold }
-          .map { |group| group_root_site(group.id, :fields => %w[id]) }.compact
+          .map { |group| group_root_site(group.id, :fields => fields) }.compact
       end
 
       def document_libraries(drives)
@@ -261,30 +237,15 @@ module ConnectorsSdk
         request_endpoint(:endpoint => "drives/#{drive_id}/root", :query_params => query_params)
       end
 
-      def item_children(drive_id, item_id, fields: [], break_after_page: false, &block)
-        next_link = cursors.delete('item_children_next_link') if break_after_page
-
-        response = if next_link.present?
-                     request_json(:url => next_link)
-                   else
-                     endpoint = "drives/#{drive_id}/items/#{item_id}/children"
-                     query_params = transform_fields_to_request_query_params(fields)
-                     request_endpoint(:endpoint => endpoint, :query_params => query_params)
-                   end
+      def item_children(drive_id, item_id, fields: [], &block)
+        endpoint = "drives/#{drive_id}/items/#{item_id}/children"
+        query_params = transform_fields_to_request_query_params(fields)
+        response = request_endpoint(:endpoint => endpoint, :query_params => query_params)
 
-        yielded = 0
         loop do
           response.value.each(&block)
           next_link = response['@odata.nextLink']
-
           break if next_link.nil?
-
-          yielded += response.value.size
-          if break_after_page && yielded >= 100
-            cursors['item_children_next_link'] = next_link
-            break
-          end
-
           response = request_json(:url => next_link)
         end
       end

data/lib/connectors_sdk/office365/extractor.rb

@@ -14,52 +14,34 @@ module ConnectorsSdk
     class Extractor < ConnectorsSdk::Base::Extractor
       DRIVE_IDS_CURSOR_KEY = 'drive_ids'.freeze
 
-      def yield_document_changes(modified_since: nil, break_after_page: false, &block)
+      def yield_document_changes(modified_since: nil, &block)
         drives_to_index.each do |drive|
           drive_id = drive.id
-
-          if break_after_page
-            current_drive_id = config.cursors['current_drive_id']
-            if current_drive_id.present? && current_drive_id > drive_id # they come alpha sorted
-              next
-            end
-            config.cursors['current_drive_id'] = drive_id
-          end
-
           drive_owner_name = drive.dig(:owner, :user, :displayName)
           drive_name = drive.name
+          site_name = drive.site_name
 
           drive_id_to_delta_link = config.cursors.fetch(DRIVE_IDS_CURSOR_KEY, {})
           begin
             if start_delta_link = drive_id_to_delta_link[drive_id]
               log_debug("Starting an incremental crawl with cursor for #{service_type.classify} with drive_id: #{drive_id}")
               begin
-                yield_changes(drive_id, :start_delta_link => start_delta_link, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+                yield_changes(drive_id, :start_delta_link => start_delta_link, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
               rescue ConnectorsSdk::Office365::CustomClient::Office365InvalidCursorsError
                 log_warn("Error listing changes with start_delta_link: #{start_delta_link}, falling back to full crawl")
-                yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+                yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
               end
             elsif modified_since.present?
               log_debug("Starting an incremental crawl using last_modified (no cursor found) for #{service_type.classify} with drive_id: #{drive_id}")
-              yield_changes(drive_id, :last_modified => modified_since, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+              yield_changes(drive_id, :last_modified => modified_since, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
             else
               log_debug("Starting a full crawl #{service_type.classify} with drive_id: #{drive_id}")
-              yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+              yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
             end
           rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
             log_warn("Error searching and listing drive #{drive_id}")
             capture_exception(e)
           end
-
-          if break_after_page && (config.cursors['page_cursor'].present? || config.cursors['item_children_next_link'].present?)
-            break
-          end
-        end
-
-        if break_after_page && config.cursors['page_cursor'].blank? && config.cursors['item_children_next_link'].blank?
-          @completed = true
-          config.overwrite_cursors!(retrieve_latest_cursors)
-          log_debug("Completed #{modified_since.nil? ? 'full' : 'incremental'} extraction")
         end
 
         nil
@@ -132,7 +114,7 @@ module ConnectorsSdk
         @existing_drive_item_ids ||= Set.new.tap do |ids|
           drives_to_index.each do |drive|
             client.list_items(drive.id) do |item|
-              ids << convert_id_to_fp_id(item.id)
+              ids << convert_id_to_es_id(item.id)
             end
           end
         end
@@ -142,7 +124,7 @@ module ConnectorsSdk
         raise NotImplementedError
       end
 
-      def convert_id_to_fp_id(_id)
+      def convert_id_to_es_id(_id)
         raise NotImplementedError
       end
 
@@ -156,11 +138,12 @@ module ConnectorsSdk
         ConnectorsShared::ExceptionTracking.capture_exception(office365_client_error, options)
       end
 
-      def yield_drive_items(drive_id, drive_owner_name:, drive_name:, break_after_page: false, &block)
-        client.list_items(drive_id, break_after_page: break_after_page) do |item|
+      def yield_drive_items(drive_id, drive_owner_name:, drive_name:, site_name:, &block)
+        client.list_items(drive_id) do |item|
           yield_single_document_change(:identifier => "Office365 change: #{item&.id} (#{Office365::Adapter::GraphItem.get_path(item)})") do
             item.drive_owner_name = drive_owner_name
             item.drive_name = drive_name
+            item.site_name = site_name
             yield_create_or_update(drive_id, item, &block)
           end
         end
@@ -170,15 +153,16 @@ module ConnectorsSdk
         if item.deleted.nil?
           yield_create_or_update(drive_id, item, &block)
         else
-          yield :delete, convert_id_to_fp_id(item.id)
+          yield :delete, convert_id_to_es_id(item.id)
         end
       end
 
-      def yield_changes(drive_id, drive_owner_name:, drive_name:, start_delta_link: nil, last_modified: nil, break_after_page: false, &block)
-        client.list_changes(:drive_id => drive_id, :start_delta_link => start_delta_link, :last_modified => last_modified, :break_after_page => break_after_page) do |item|
+      def yield_changes(drive_id, drive_owner_name:, drive_name:, site_name:, start_delta_link: nil, last_modified: nil, &block)
+        client.list_changes(:drive_id => drive_id, :start_delta_link => start_delta_link, :last_modified => last_modified) do |item|
           yield_single_document_change(:identifier => "Office365 change: #{item&.id} (#{Office365::Adapter::GraphItem.get_path(item)})") do
             item.drive_owner_name = drive_owner_name
             item.drive_name = drive_name
+            item.site_name = site_name
             yield_correct_actions_and_converted_item(drive_id, item, &block)
           end
         end
@@ -210,11 +194,11 @@ module ConnectorsSdk
 
       def generate_document(item)
         if item.file
-          adapter.swiftype_document_from_file(item)
+          adapter.es_document_from_file(item)
         elsif item.folder
-          adapter.swiftype_document_from_folder(item)
+          adapter.es_document_from_folder(item)
         elsif item.package
-          adapter.swiftype_document_from_package(item)
+          adapter.es_document_from_package(item)
         else
           raise "Unexpected Office 365 item type for item #{item}"
         end

data/lib/connectors_sdk/share_point/adapter.rb

@@ -10,36 +10,48 @@ require 'connectors_sdk/office365/adapter'
 
 module ConnectorsSdk
   module SharePoint
+    module SitePrefix
+      def get_path(item)
+        item.site_name.present? ? "/sites/#{item.site_name}#{super}" : super
+      end
+    end
+
     class Adapter < Office365::Adapter
       generate_id_helpers :share_point, 'share_point'
 
-      def self.swiftype_document_from_file(file)
-        FileGraphItem.new(file).to_swiftype_document
+      def self.es_document_from_file(file)
+        FileGraphItem.new(file).to_es_document
       end
 
-      def self.swiftype_document_from_folder(folder)
-        FolderGraphItem.new(folder).to_swiftype_document
+      def self.es_document_from_folder(folder)
+        FolderGraphItem.new(folder).to_es_document
       end
 
-      def self.swiftype_document_from_package(package)
-        PackageGraphItem.new(package).to_swiftype_document
+      def self.es_document_from_package(package)
+        PackageGraphItem.new(package).to_es_document
       end
 
       class FileGraphItem < Office365::Adapter::FileGraphItem
-        def self.convert_id_to_fp_id(id)
-          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+        include SitePrefix
+
+        def self.convert_id_to_es_id(id)
+          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end
       end
 
       class FolderGraphItem < Office365::Adapter::FolderGraphItem
-        def self.convert_id_to_fp_id(id)
-          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+        include SitePrefix
+
+        def self.convert_id_to_es_id(id)
+          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end
       end
 
       class PackageGraphItem < Office365::Adapter::PackageGraphItem
-        def self.convert_id_to_fp_id(id)
-          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+        include SitePrefix
+
+        def self.convert_id_to_es_id(id)
+          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end
       end
     end

data/lib/connectors_sdk/share_point/authorization.rb

@@ -6,70 +6,12 @@
 
 # frozen_string_literal: true
 
-require 'connectors_shared'
-require 'signet'
-require 'signet/oauth_2'
-require 'signet/oauth_2/client'
+require 'connectors_sdk/base/authorization'
 
 module ConnectorsSdk
   module SharePoint
-    class Authorization
+    class Authorization < ConnectorsSdk::Base::Authorization
       class << self
-        def authorization_url
-          'https://login.microsoftonline.com/common/oauth2/v2.0/authorize'
-        end
-
-        def token_credential_uri
-          'https://login.microsoftonline.com/common/oauth2/v2.0/token'
-        end
-
-        def authorization_uri(params)
-          missing = missing_fields(params, %w[client_id])
-          unless missing.blank?
-            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
-          end
-
-          params[:response_type] = 'code'
-          params[:additional_parameters] = { :prompt => 'consent' }
-          client = oauth_client(params)
-          client.authorization_uri.to_s
-        end
-
-        def access_token(params)
-          missing = missing_fields(params, %w[client_id client_secret code redirect_uri])
-          unless missing.blank?
-            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
-          end
-
-          params[:grant_type] = 'authorization_code'
-          client = oauth_client(params)
-          client.fetch_access_token
-        end
-
-        def refresh(params)
-          missing = missing_fields(params, %w[client_id client_secret refresh_token])
-          unless missing.blank?
-            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
-          end
-
-          params[:grant_type] = 'refresh_token'
-          client = oauth_client(params)
-          client.refresh!
-        rescue StandardError => e
-          ConnectorsShared::ExceptionTracking.log_exception(e)
-          raise ConnectorsShared::TokenRefreshFailedError
-        end
-
-        def oauth_client(params)
-          options = params.merge(
-            :authorization_uri => authorization_url,
-            :token_credential_uri => token_credential_uri,
-            :scope => oauth_scope
-          )
-          options[:state] = JSON.dump(options[:state]) if options[:state]
-          Signet::OAuth2::Client.new(options)
-        end
-
         def oauth_scope
           %w[
             User.ReadBasic.All
@@ -82,8 +24,18 @@ module ConnectorsSdk
           ]
         end
 
-        def missing_fields(params, required = [])
-          Array.wrap(required).select { |field| params[field.to_sym].nil? }
+        private
+
+        def authorization_url
+          'https://login.microsoftonline.com/common/oauth2/v2.0/authorize'
+        end
+
+        def token_credential_uri
+          'https://login.microsoftonline.com/common/oauth2/v2.0/token'
+        end
+
+        def additional_parameters
+          { :prompt => 'consent' }
         end
       end
     end

data/lib/connectors_sdk/share_point/extractor.rb

@@ -15,8 +15,8 @@ module ConnectorsSdk
 
       private
 
-      def convert_id_to_fp_id(id)
-        ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+      def convert_id_to_es_id(id)
+        ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
       end
 
       def adapter