connectors_sdk 8.3.0.0.pre.20220510T144908Z → 8.3.0.0.pre.20220517T144653Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 94b75b9fa3a5f0c46a271a34d073f71977629db6095b6364c8710b0aab92374b
-  data.tar.gz: d133f34052f43e0b8b65ba10eb946c55a6ef421a6296d2d4438e7f4f92b45696
+  metadata.gz: b86ca5e489e3cef3b9f2c04a462baf71d6b43805731b0cb52ba2f56f5078d6d3
+  data.tar.gz: 044e860f11163e82c63f66276c3d5628b761c5dfcc5168fc8f80b63ca87d19f0
 SHA512:
-  metadata.gz: 712b819efcfa755ce19e4cdb7060dbadefecf37321d07bfe4e7bc9b18254e0f11f5fc29967ff171b1314064d77427bde730ab400a4a2dcde89b31dc6344d4a34
-  data.tar.gz: fa1fedb0c7449b9b2b50b1a56a710178cd3343a108500938244987eb31c694ab0d0214f383df69b6efa6b384eb558697553813c9b93cb142f011b374e05f2c49
+  metadata.gz: 528fa5260cf80a3ebb918478e1be2e7cac1668588853ad72b1259095086090c15a0cbc028523b1de8b775b3b64ed7427c6fab7ffccc562d559c8164456b84c4b
+  data.tar.gz: a58d353e2b48ffda33aa287d0fa6bb1400c531aae2794995ff3b644b779426d10bb3efc8735d8e07c353fae64bc0b3b0af58427d989f499210a19244b23c1a35

data/lib/connectors_sdk/base/custom_client.rb

@@ -74,7 +74,7 @@ module ConnectorsSdk
             faraday.use(*middleware_config)
           end
 
-          faraday.adapter(:httpclient)
+          faraday.adapter :httpclient
         end
       end
 

data/lib/connectors_sdk/base/http_call_wrapper.rb

@@ -22,31 +22,31 @@ module ConnectorsSdk
         )
       end
 
-      def document_batch(params)
+      def extract(params)
         convert_third_party_errors do
-          results = []
-
           extractor = extractor(params)
 
-          extractor.yield_document_changes(:break_after_page => true, :modified_since => extractor.config.cursors['modified_since']) do |action, doc, download_args_and_proc|
+          extractor.yield_document_changes(:modified_since => extractor.config.cursors[:modified_since]) do |action, doc, download_args_and_proc|
             download_obj = nil
             if download_args_and_proc
               download_obj = {
-                  id: download_args_and_proc[0],
-                  name: download_args_and_proc[1],
-                  size: download_args_and_proc[2],
-                  download_args: download_args_and_proc[3]
+                id: download_args_and_proc[0],
+                name: download_args_and_proc[1],
+                size: download_args_and_proc[2],
+                download_args: download_args_and_proc[3]
               }
             end
 
-            results << {
-                :action => action,
-                :document => doc,
-                :download => download_obj
+            doc = {
+              :action => action,
+              :document => doc,
+              :download => download_obj
             }
+
+            yield doc
           end
 
-          [results, extractor.config.cursors, extractor.completed]
+          extractor.config.to_h[:cursors]
         end
       end
 

data/lib/connectors_sdk/confluence/extractor.rb

@@ -20,7 +20,7 @@ module ConnectorsSdk
 
       ConnectorsSdk::Base::Extractor::TRANSIENT_SERVER_ERROR_CLASSES << Atlassian::CustomClient::ServiceUnavailableError
 
-      def yield_document_changes(modified_since: nil, break_after_page: false)
+      def yield_document_changes(modified_since: nil)
         @space_permissions_cache = {}
         @content_restriction_cache = {}
         yield_spaces do |space|
@@ -50,11 +50,6 @@ module ConnectorsSdk
               yield :create_or_update, Confluence::Adapter.es_document_from_confluence_content(content, content_base_url, restrictions)
             end
           end
-
-          if break_after_page
-            @completed = true
-            break
-          end
         end
       end
 

data/lib/connectors_sdk/office365/custom_client.rb

@@ -52,7 +52,6 @@ module ConnectorsSdk
       def initialize(access_token:, cursors: {}, ensure_fresh_auth: nil)
         @access_token = access_token
         @cursors = cursors || {}
-        @cursors[ConnectorsSdk::Office365::Extractor::DRIVE_IDS_CURSOR_KEY] ||= {}
         super(:ensure_fresh_auth => ensure_fresh_auth)
       end
 
@@ -77,12 +76,11 @@ module ConnectorsSdk
         # recently created groups (new Private Team site will be there) to reduce friction and index this site
         # earlier.
         # See: https://github.com/elastic/ent-search/pull/3581
-        share_point_sites = (sites(:fields => %w[id]) + recent_share_point_group_sites(:fields => %[id]))
+        share_point_sites = (sites(:fields => %w[id,name]) + recent_share_point_group_sites(:fields => %w[id,name]))
 
         share_point_sites
-          .map(&:id)
-          .uniq
-          .map { |site_id| site_drives(site_id, :fields => fields) }
+          .uniq(&:id)
+          .map { |site| site_drives(site, :fields => fields) }
           .flatten
           .compact
       end
@@ -104,47 +102,32 @@ module ConnectorsSdk
         request_all(:endpoint => 'sites/', :fields => fields, :additional_query_params => { :search => '', :top => 10 })
       end
 
-      def site_drives(site_id, fields: [])
+      def site_drives(site, fields: [])
         document_libraries(
-          request_all(:endpoint => "sites/#{site_id}/drives/", :fields => fields)
-        )
+          request_all(:endpoint => "sites/#{site.id}/drives/", :fields => fields)
+        ).map do |drive|
+          drive.site_name = site.name
+          drive
+        end
       rescue ClientError => e
         ConnectorsShared::Logger.info("Received response of #{e.status_code} trying to get drive for Site with Id = #{site_id}: #{e.message}")
         nil
       end
 
-      def list_items(drive_id, fields: [], break_after_page: false)
+      def list_items(drive_id, fields: [])
         # MSFT Graph API does not have a recursive list items, have to do this dfs style
-
-        stack = if break_after_page && cursors['page_cursor'].present?
-                  cursors.delete('page_cursor')
-                else
-                  [get_root_item(drive_id, ['id']).id]
-                end
-
+        stack = [get_root_item(drive_id, ['id']).id]
         # We rely on the id field below to perform our DFS
         fields_with_id = fields.any? ? fields | ['id'] : fields
-        yielded = 0
         while stack.any?
           folder_id = stack.pop
-          item_children(drive_id, folder_id, :fields => fields_with_id, :break_after_page => break_after_page) do |item|
+          item_children(drive_id, folder_id, :fields => fields_with_id) do |item|
             if item.folder
               stack << item.id
             end
             yield item
-
-            yielded += 1
           end
 
-          if break_after_page && yielded >= 100
-            if cursors['item_children_next_link'].present?
-              stack << folder_id
-            end
-            if stack.any?
-              cursors['page_cursor'] = stack.dup
-              break
-            end
-          end
         end
       end
 
@@ -152,19 +135,16 @@ module ConnectorsSdk
         request_endpoint(:endpoint => "drives/#{drive_id}/items/#{item_id}/permissions").value
       end
 
-      def list_changes(drive_id:, start_delta_link: nil, last_modified: nil, break_after_page: false)
+      def list_changes(drive_id:, start_delta_link: nil, last_modified: nil)
         query_params = { :'$select' => %w(id content.downloadUrl lastModifiedDateTime lastModifiedBy root deleted file folder package name webUrl createdBy createdDateTime size).join(',') }
         response =
-          if break_after_page && cursors['page_cursor'].present?
-            request_json(:url => cursors.delete('page_cursor'))
-          elsif start_delta_link.nil?
+          if start_delta_link.nil?
             endpoint = "drives/#{drive_id}/root/delta"
             request_endpoint(:endpoint => endpoint, :query_params => query_params)
           else
             request_json(:url => start_delta_link, :query_params => query_params)
           end
 
-        yielded = 0
         loop do
           response.value.each do |change|
             # MSFT Graph API does not allow us to view "changes" in chronological order, so if there is no cursor,
@@ -172,25 +152,18 @@ module ConnectorsSdk
             # since to get another cursor, we would have to go through all the changes anyway
             next if last_modified.present? && Time.parse(change.lastModifiedDateTime) < last_modified
             next if change.root # We don't want to index the root of the drive
-
             yield change
-            yielded += 1
-          end
-
-          if break_after_page && yielded >= 100 && response['@odata.nextLink'].present?
-            cursors['page_cursor'] = response['@odata.nextLink']
-            break
           end
 
           break if response['@odata.nextLink'].nil?
           response = request_json(:url => response['@odata.nextLink'])
         end
 
-        cursors[ConnectorsSdk::Office365::Extractor::DRIVE_IDS_CURSOR_KEY][drive_id] = response['@odata.deltaLink']
+        cursors[drive_id] = response['@odata.deltaLink']
       end
 
       def get_latest_delta_link(drive_id)
-        cursors[ConnectorsSdk::Office365::Extractor::DRIVE_IDS_CURSOR_KEY][drive_id] || exhaustively_get_delta_link(drive_id)
+        cursors[drive_id] || exhaustively_get_delta_link(drive_id)
       end
 
       def exhaustively_get_delta_link(drive_id)
@@ -210,6 +183,7 @@ module ConnectorsSdk
       def download_item(download_url)
         request(:url => download_url) do |request|
           request.options.params_encoder = Office365DownloadParamsEncoder
+          request.options.timeout = 30
         end.body
       end
 
@@ -235,7 +209,7 @@ module ConnectorsSdk
 
         groups(:fields => %w(id createdDateTime))
           .select { |group| group.createdDateTime > created_date_time_threshold }
-          .map { |group| group_root_site(group.id, :fields => %w[id]) }.compact
+          .map { |group| group_root_site(group.id, :fields => fields) }.compact
       end
 
       def document_libraries(drives)
@@ -263,30 +237,15 @@ module ConnectorsSdk
         request_endpoint(:endpoint => "drives/#{drive_id}/root", :query_params => query_params)
       end
 
-      def item_children(drive_id, item_id, fields: [], break_after_page: false, &block)
-        next_link = cursors.delete('item_children_next_link') if break_after_page
-
-        response = if next_link.present?
-                     request_json(:url => next_link)
-                   else
-                     endpoint = "drives/#{drive_id}/items/#{item_id}/children"
-                     query_params = transform_fields_to_request_query_params(fields)
-                     request_endpoint(:endpoint => endpoint, :query_params => query_params)
-                   end
+      def item_children(drive_id, item_id, fields: [], &block)
+        endpoint = "drives/#{drive_id}/items/#{item_id}/children"
+        query_params = transform_fields_to_request_query_params(fields)
+        response = request_endpoint(:endpoint => endpoint, :query_params => query_params)
 
-        yielded = 0
         loop do
           response.value.each(&block)
           next_link = response['@odata.nextLink']
-
           break if next_link.nil?
-
-          yielded += response.value.size
-          if break_after_page && yielded >= 100
-            cursors['item_children_next_link'] = next_link
-            break
-          end
-
           response = request_json(:url => next_link)
         end
       end

data/lib/connectors_sdk/office365/extractor.rb

@@ -14,52 +14,34 @@ module ConnectorsSdk
     class Extractor < ConnectorsSdk::Base::Extractor
       DRIVE_IDS_CURSOR_KEY = 'drive_ids'.freeze
 
-      def yield_document_changes(modified_since: nil, break_after_page: false, &block)
+      def yield_document_changes(modified_since: nil, &block)
         drives_to_index.each do |drive|
           drive_id = drive.id
-
-          if break_after_page
-            current_drive_id = config.cursors['current_drive_id']
-            if current_drive_id.present? && current_drive_id > drive_id # they come alpha sorted
-              next
-            end
-            config.cursors['current_drive_id'] = drive_id
-          end
-
           drive_owner_name = drive.dig(:owner, :user, :displayName)
           drive_name = drive.name
+          site_name = drive.site_name
 
           drive_id_to_delta_link = config.cursors.fetch(DRIVE_IDS_CURSOR_KEY, {})
           begin
             if start_delta_link = drive_id_to_delta_link[drive_id]
               log_debug("Starting an incremental crawl with cursor for #{service_type.classify} with drive_id: #{drive_id}")
               begin
-                yield_changes(drive_id, :start_delta_link => start_delta_link, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+                yield_changes(drive_id, :start_delta_link => start_delta_link, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
               rescue ConnectorsSdk::Office365::CustomClient::Office365InvalidCursorsError
                 log_warn("Error listing changes with start_delta_link: #{start_delta_link}, falling back to full crawl")
-                yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+                yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
               end
             elsif modified_since.present?
               log_debug("Starting an incremental crawl using last_modified (no cursor found) for #{service_type.classify} with drive_id: #{drive_id}")
-              yield_changes(drive_id, :last_modified => modified_since, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+              yield_changes(drive_id, :last_modified => modified_since, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
             else
               log_debug("Starting a full crawl #{service_type.classify} with drive_id: #{drive_id}")
-              yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+              yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
             end
           rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
             log_warn("Error searching and listing drive #{drive_id}")
             capture_exception(e)
           end
-
-          if break_after_page && (config.cursors['page_cursor'].present? || config.cursors['item_children_next_link'].present?)
-            break
-          end
-        end
-
-        if break_after_page && config.cursors['page_cursor'].blank? && config.cursors['item_children_next_link'].blank?
-          @completed = true
-          config.overwrite_cursors!(retrieve_latest_cursors)
-          log_debug("Completed #{modified_since.nil? ? 'full' : 'incremental'} extraction")
         end
 
         nil
@@ -156,11 +138,12 @@ module ConnectorsSdk
         ConnectorsShared::ExceptionTracking.capture_exception(office365_client_error, options)
       end
 
-      def yield_drive_items(drive_id, drive_owner_name:, drive_name:, break_after_page: false, &block)
-        client.list_items(drive_id, break_after_page: break_after_page) do |item|
+      def yield_drive_items(drive_id, drive_owner_name:, drive_name:, site_name:, &block)
+        client.list_items(drive_id) do |item|
           yield_single_document_change(:identifier => "Office365 change: #{item&.id} (#{Office365::Adapter::GraphItem.get_path(item)})") do
             item.drive_owner_name = drive_owner_name
             item.drive_name = drive_name
+            item.site_name = site_name
             yield_create_or_update(drive_id, item, &block)
           end
         end
@@ -174,11 +157,12 @@ module ConnectorsSdk
         end
       end
 
-      def yield_changes(drive_id, drive_owner_name:, drive_name:, start_delta_link: nil, last_modified: nil, break_after_page: false, &block)
-        client.list_changes(:drive_id => drive_id, :start_delta_link => start_delta_link, :last_modified => last_modified, :break_after_page => break_after_page) do |item|
+      def yield_changes(drive_id, drive_owner_name:, drive_name:, site_name:, start_delta_link: nil, last_modified: nil, &block)
+        client.list_changes(:drive_id => drive_id, :start_delta_link => start_delta_link, :last_modified => last_modified) do |item|
           yield_single_document_change(:identifier => "Office365 change: #{item&.id} (#{Office365::Adapter::GraphItem.get_path(item)})") do
             item.drive_owner_name = drive_owner_name
             item.drive_name = drive_name
+            item.site_name = site_name
             yield_correct_actions_and_converted_item(drive_id, item, &block)
           end
         end

data/lib/connectors_sdk/share_point/adapter.rb

@@ -10,6 +10,12 @@ require 'connectors_sdk/office365/adapter'
 
 module ConnectorsSdk
   module SharePoint
+    module SitePrefix
+      def get_path(item)
+        item.site_name.present? ? "/sites/#{item.site_name}#{super}" : super
+      end
+    end
+
     class Adapter < Office365::Adapter
       generate_id_helpers :share_point, 'share_point'
 
@@ -26,18 +32,24 @@ module ConnectorsSdk
       end
 
       class FileGraphItem < Office365::Adapter::FileGraphItem
+        include SitePrefix
+
         def self.convert_id_to_es_id(id)
           ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end
       end
 
       class FolderGraphItem < Office365::Adapter::FolderGraphItem
+        include SitePrefix
+
         def self.convert_id_to_es_id(id)
           ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end
       end
 
       class PackageGraphItem < Office365::Adapter::PackageGraphItem
+        include SitePrefix
+
         def self.convert_id_to_es_id(id)
           ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end

data/lib/connectors_shared/job_status.rb

@@ -0,0 +1,18 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+module ConnectorsShared
+  class JobStatus
+    CREATED = 'created'
+    RUNNING = 'running'
+    FINISHED = 'finished'
+    FAILED = 'failed'
+
+    def self.is_valid?(status)
+      [CREATED, RUNNING, FINISHED, FAILED].include? status
+    end
+  end
+end

data/lib/connectors_shared.rb

@@ -8,5 +8,6 @@ require 'connectors_shared/constants'
 require 'connectors_shared/errors'
 require 'connectors_shared/exception_tracking'
 require 'connectors_shared/extension_mapping_util'
+require 'connectors_shared/job_status'
 require 'connectors_shared/logger'
 require 'connectors_shared/monitor'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_sdk
 version: !ruby/object:Gem::Version
-  version: 8.3.0.0.pre.20220510T144908Z
+  version: 8.3.0.0.pre.20220517T144653Z
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-10 00:00:00.000000000 Z
+date: 2022-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -92,6 +92,7 @@ files:
 - lib/connectors_shared/exception_tracking.rb
 - lib/connectors_shared/extension_mapping_util.rb
 - lib/connectors_shared/extraction_utils.rb
+- lib/connectors_shared/job_status.rb
 - lib/connectors_shared/logger.rb
 - lib/connectors_shared/middleware/basic_auth.rb
 - lib/connectors_shared/middleware/bearer_auth.rb
@@ -102,7 +103,7 @@ homepage: https://github.com/elastic/connectors
 licenses:
 - Elastic-2.0
 metadata:
-  revision: fb1187beef857b555633e1804eef3ed5e586091d
+  revision: 9f25f35e17ffb36dfda754d657794ed9b5d2d75a
   repository: git@github.com:elastic/connectors.git
 post_install_message:
 rdoc_options: []