connectors_sdk 8.3.0.0.pre.20220510T144908Z → 8.3.2.0

data/lib/connectors_sdk/gitlab/extractor.rb

@@ -0,0 +1,123 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/extractor'
+require 'connectors_sdk/gitlab/custom_client'
+require 'connectors_sdk/gitlab/adapter'
+require 'connectors_sdk/gitlab/config'
+require 'rack/utils'
+
+module ConnectorsSdk
+  module GitLab
+    class Extractor < ConnectorsSdk::Base::Extractor
+      PAGE_SIZE = 100 # max is 100
+
+      def yield_document_changes(modified_since: nil)
+        query_params = {
+          :pagination => :keyset,
+          :per_page => PAGE_SIZE,
+          :order_by => :id,
+          :sort => :desc
+        }
+        # looks like it's an incremental sync
+        if modified_since.present?
+          date_since = modified_since.is_a?(Time) ? modified_since : Time.new(modified_since)
+          query_params[:last_activity_after] = date_since.iso8601
+        end
+
+        next_page_link = nil
+
+        loop do
+          if next_page_link.present?
+            if (matcher = /(https?:[^>]*)/.match(next_page_link))
+              clean_query = URI.parse(matcher.captures[0]).query
+              query_params = Rack::Utils.parse_query(clean_query)
+            else
+              raise "Next page link has unexpected format: #{next_page_link}"
+            end
+          end
+          response = client.get('projects', query_params)
+
+          JSON.parse(response.body).map do |doc|
+            doc = doc.with_indifferent_access
+            if config.index_permissions
+              doc = doc.merge(project_permissions(doc[:id], doc[:visibility]))
+            end
+            yield :create_or_update, ConnectorsSdk::GitLab::Adapter.to_es_document(:project, doc), nil
+          end
+
+          next_page_link = response.headers['Link'] || nil
+          break unless next_page_link.present?
+        end
+      end
+
+      def yield_deleted_ids(ids)
+        if ids.present?
+          ids.each do |id|
+            response = client.get("projects/#{id}")
+            if response.status == 404
+              # not found - assume deleted
+              yield id
+            else
+              unless response.success?
+                raise "Could not get a project by ID: #{id}, response code: #{response.status}, response: #{response.body}"
+              end
+            end
+          end
+        end
+      end
+
+      def yield_permissions(source_user_id)
+        result = []
+        if source_user_id.present?
+          result.push("user:#{source_user_id}")
+
+          user_response = client.get("users/#{source_user_id}")
+          if user_response.success?
+            username = JSON.parse(user_response.body).with_indifferent_access[:username]
+            query = { :external => true, :username => username }
+            external_response = client.get('users', query)
+            if external_response.success?
+              external_users = Hashie::Array.new(JSON.parse(external_response.body))
+              if external_users.empty?
+                # the user is not external
+                result.push('type:internal')
+              end
+            else
+              raise "Could not check external user status by ID: #{source_user_id}"
+            end
+          else
+            raise "User isn't found by ID: #{source_user_id}"
+          end
+        end
+        yield result
+      end
+
+      private
+
+      def project_permissions(id, visibility)
+        result = []
+        if visibility.to_sym == :public || !config.index_permissions
+          # visible-to-all
+          return {}
+        end
+        if visibility.to_sym == :internal
+          result.push('type:internal')
+        end
+        response = client.get("projects/#{id}/members/all")
+        if response.success?
+          members = Hashie::Array.new(JSON.parse(response.body))
+          result.concat(members.map { |user| "user:#{user[:id]}" })
+        else
+          raise "Could not get project members by project ID: #{id}, response code: #{response.status}, response: #{response.body}"
+        end
+        { :_allow_permissions => result }
+      end
+    end
+  end
+end

data/lib/connectors_sdk/office365/custom_client.rb

@@ -52,7 +52,6 @@ module ConnectorsSdk
       def initialize(access_token:, cursors: {}, ensure_fresh_auth: nil)
         @access_token = access_token
         @cursors = cursors || {}
-        @cursors[ConnectorsSdk::Office365::Extractor::DRIVE_IDS_CURSOR_KEY] ||= {}
         super(:ensure_fresh_auth => ensure_fresh_auth)
       end
 
@@ -77,12 +76,11 @@ module ConnectorsSdk
         # recently created groups (new Private Team site will be there) to reduce friction and index this site
         # earlier.
         # See: https://github.com/elastic/ent-search/pull/3581
-        share_point_sites = (sites(:fields => %w[id]) + recent_share_point_group_sites(:fields => %[id]))
+        share_point_sites = (sites(:fields => %w[id,name]) + recent_share_point_group_sites(:fields => %w[id,name]))
 
         share_point_sites
-          .map(&:id)
-          .uniq
-          .map { |site_id| site_drives(site_id, :fields => fields) }
+          .uniq(&:id)
+          .map { |site| site_drives(site, :fields => fields) }
           .flatten
           .compact
       end
@@ -104,47 +102,32 @@ module ConnectorsSdk
         request_all(:endpoint => 'sites/', :fields => fields, :additional_query_params => { :search => '', :top => 10 })
       end
 
-      def site_drives(site_id, fields: [])
+      def site_drives(site, fields: [])
         document_libraries(
-          request_all(:endpoint => "sites/#{site_id}/drives/", :fields => fields)
-        )
+          request_all(:endpoint => "sites/#{site.id}/drives/", :fields => fields)
+        ).map do |drive|
+          drive.site_name = site.name
+          drive
+        end
       rescue ClientError => e
         ConnectorsShared::Logger.info("Received response of #{e.status_code} trying to get drive for Site with Id = #{site_id}: #{e.message}")
         nil
       end
 
-      def list_items(drive_id, fields: [], break_after_page: false)
+      def list_items(drive_id, fields: [])
         # MSFT Graph API does not have a recursive list items, have to do this dfs style
-
-        stack = if break_after_page && cursors['page_cursor'].present?
-                  cursors.delete('page_cursor')
-                else
-                  [get_root_item(drive_id, ['id']).id]
-                end
-
+        stack = [get_root_item(drive_id, ['id']).id]
         # We rely on the id field below to perform our DFS
         fields_with_id = fields.any? ? fields | ['id'] : fields
-        yielded = 0
         while stack.any?
           folder_id = stack.pop
-          item_children(drive_id, folder_id, :fields => fields_with_id, :break_after_page => break_after_page) do |item|
+          item_children(drive_id, folder_id, :fields => fields_with_id) do |item|
             if item.folder
               stack << item.id
             end
             yield item
-
-            yielded += 1
           end
 
-          if break_after_page && yielded >= 100
-            if cursors['item_children_next_link'].present?
-              stack << folder_id
-            end
-            if stack.any?
-              cursors['page_cursor'] = stack.dup
-              break
-            end
-          end
         end
       end
 
@@ -152,19 +135,16 @@ module ConnectorsSdk
         request_endpoint(:endpoint => "drives/#{drive_id}/items/#{item_id}/permissions").value
       end
 
-      def list_changes(drive_id:, start_delta_link: nil, last_modified: nil, break_after_page: false)
+      def list_changes(drive_id:, start_delta_link: nil, last_modified: nil)
         query_params = { :'$select' => %w(id content.downloadUrl lastModifiedDateTime lastModifiedBy root deleted file folder package name webUrl createdBy createdDateTime size).join(',') }
         response =
-          if break_after_page && cursors['page_cursor'].present?
-            request_json(:url => cursors.delete('page_cursor'))
-          elsif start_delta_link.nil?
+          if start_delta_link.nil?
             endpoint = "drives/#{drive_id}/root/delta"
             request_endpoint(:endpoint => endpoint, :query_params => query_params)
           else
             request_json(:url => start_delta_link, :query_params => query_params)
           end
 
-        yielded = 0
         loop do
           response.value.each do |change|
             # MSFT Graph API does not allow us to view "changes" in chronological order, so if there is no cursor,
@@ -172,25 +152,18 @@ module ConnectorsSdk
             # since to get another cursor, we would have to go through all the changes anyway
             next if last_modified.present? && Time.parse(change.lastModifiedDateTime) < last_modified
             next if change.root # We don't want to index the root of the drive
-
             yield change
-            yielded += 1
-          end
-
-          if break_after_page && yielded >= 100 && response['@odata.nextLink'].present?
-            cursors['page_cursor'] = response['@odata.nextLink']
-            break
           end
 
           break if response['@odata.nextLink'].nil?
           response = request_json(:url => response['@odata.nextLink'])
         end
 
-        cursors[ConnectorsSdk::Office365::Extractor::DRIVE_IDS_CURSOR_KEY][drive_id] = response['@odata.deltaLink']
+        cursors[drive_id] = response['@odata.deltaLink']
       end
 
       def get_latest_delta_link(drive_id)
-        cursors[ConnectorsSdk::Office365::Extractor::DRIVE_IDS_CURSOR_KEY][drive_id] || exhaustively_get_delta_link(drive_id)
+        cursors[drive_id] || exhaustively_get_delta_link(drive_id)
       end
 
       def exhaustively_get_delta_link(drive_id)
@@ -210,6 +183,7 @@ module ConnectorsSdk
       def download_item(download_url)
         request(:url => download_url) do |request|
           request.options.params_encoder = Office365DownloadParamsEncoder
+          request.options.timeout = 30
         end.body
       end
 
@@ -235,7 +209,7 @@ module ConnectorsSdk
 
         groups(:fields => %w(id createdDateTime))
           .select { |group| group.createdDateTime > created_date_time_threshold }
-          .map { |group| group_root_site(group.id, :fields => %w[id]) }.compact
+          .map { |group| group_root_site(group.id, :fields => fields) }.compact
       end
 
       def document_libraries(drives)
@@ -263,30 +237,15 @@ module ConnectorsSdk
         request_endpoint(:endpoint => "drives/#{drive_id}/root", :query_params => query_params)
       end
 
-      def item_children(drive_id, item_id, fields: [], break_after_page: false, &block)
-        next_link = cursors.delete('item_children_next_link') if break_after_page
-
-        response = if next_link.present?
-                     request_json(:url => next_link)
-                   else
-                     endpoint = "drives/#{drive_id}/items/#{item_id}/children"
-                     query_params = transform_fields_to_request_query_params(fields)
-                     request_endpoint(:endpoint => endpoint, :query_params => query_params)
-                   end
+      def item_children(drive_id, item_id, fields: [], &block)
+        endpoint = "drives/#{drive_id}/items/#{item_id}/children"
+        query_params = transform_fields_to_request_query_params(fields)
+        response = request_endpoint(:endpoint => endpoint, :query_params => query_params)
 
-        yielded = 0
         loop do
           response.value.each(&block)
           next_link = response['@odata.nextLink']
-
           break if next_link.nil?
-
-          yielded += response.value.size
-          if break_after_page && yielded >= 100
-            cursors['item_children_next_link'] = next_link
-            break
-          end
-
           response = request_json(:url => next_link)
         end
       end

data/lib/connectors_sdk/office365/extractor.rb

@@ -14,52 +14,34 @@ module ConnectorsSdk
     class Extractor < ConnectorsSdk::Base::Extractor
       DRIVE_IDS_CURSOR_KEY = 'drive_ids'.freeze
 
-      def yield_document_changes(modified_since: nil, break_after_page: false, &block)
+      def yield_document_changes(modified_since: nil, &block)
         drives_to_index.each do |drive|
           drive_id = drive.id
-
-          if break_after_page
-            current_drive_id = config.cursors['current_drive_id']
-            if current_drive_id.present? && current_drive_id > drive_id # they come alpha sorted
-              next
-            end
-            config.cursors['current_drive_id'] = drive_id
-          end
-
           drive_owner_name = drive.dig(:owner, :user, :displayName)
           drive_name = drive.name
+          site_name = drive.site_name
 
           drive_id_to_delta_link = config.cursors.fetch(DRIVE_IDS_CURSOR_KEY, {})
           begin
             if start_delta_link = drive_id_to_delta_link[drive_id]
               log_debug("Starting an incremental crawl with cursor for #{service_type.classify} with drive_id: #{drive_id}")
               begin
-                yield_changes(drive_id, :start_delta_link => start_delta_link, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+                yield_changes(drive_id, :start_delta_link => start_delta_link, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
               rescue ConnectorsSdk::Office365::CustomClient::Office365InvalidCursorsError
                 log_warn("Error listing changes with start_delta_link: #{start_delta_link}, falling back to full crawl")
-                yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+                yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
               end
             elsif modified_since.present?
               log_debug("Starting an incremental crawl using last_modified (no cursor found) for #{service_type.classify} with drive_id: #{drive_id}")
-              yield_changes(drive_id, :last_modified => modified_since, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+              yield_changes(drive_id, :last_modified => modified_since, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
             else
               log_debug("Starting a full crawl #{service_type.classify} with drive_id: #{drive_id}")
-              yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+              yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :site_name => site_name, &block)
             end
           rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
             log_warn("Error searching and listing drive #{drive_id}")
             capture_exception(e)
           end
-
-          if break_after_page && (config.cursors['page_cursor'].present? || config.cursors['item_children_next_link'].present?)
-            break
-          end
-        end
-
-        if break_after_page && config.cursors['page_cursor'].blank? && config.cursors['item_children_next_link'].blank?
-          @completed = true
-          config.overwrite_cursors!(retrieve_latest_cursors)
-          log_debug("Completed #{modified_since.nil? ? 'full' : 'incremental'} extraction")
         end
 
         nil
@@ -156,11 +138,12 @@ module ConnectorsSdk
         ConnectorsShared::ExceptionTracking.capture_exception(office365_client_error, options)
       end
 
-      def yield_drive_items(drive_id, drive_owner_name:, drive_name:, break_after_page: false, &block)
-        client.list_items(drive_id, break_after_page: break_after_page) do |item|
+      def yield_drive_items(drive_id, drive_owner_name:, drive_name:, site_name:, &block)
+        client.list_items(drive_id) do |item|
           yield_single_document_change(:identifier => "Office365 change: #{item&.id} (#{Office365::Adapter::GraphItem.get_path(item)})") do
             item.drive_owner_name = drive_owner_name
             item.drive_name = drive_name
+            item.site_name = site_name
             yield_create_or_update(drive_id, item, &block)
           end
         end
@@ -174,11 +157,12 @@ module ConnectorsSdk
         end
       end
 
-      def yield_changes(drive_id, drive_owner_name:, drive_name:, start_delta_link: nil, last_modified: nil, break_after_page: false, &block)
-        client.list_changes(:drive_id => drive_id, :start_delta_link => start_delta_link, :last_modified => last_modified, :break_after_page => break_after_page) do |item|
+      def yield_changes(drive_id, drive_owner_name:, drive_name:, site_name:, start_delta_link: nil, last_modified: nil, &block)
+        client.list_changes(:drive_id => drive_id, :start_delta_link => start_delta_link, :last_modified => last_modified) do |item|
           yield_single_document_change(:identifier => "Office365 change: #{item&.id} (#{Office365::Adapter::GraphItem.get_path(item)})") do
             item.drive_owner_name = drive_owner_name
             item.drive_name = drive_name
+            item.site_name = site_name
             yield_correct_actions_and_converted_item(drive_id, item, &block)
           end
         end

data/lib/connectors_sdk/share_point/adapter.rb

@@ -10,6 +10,12 @@ require 'connectors_sdk/office365/adapter'
 
 module ConnectorsSdk
   module SharePoint
+    module SitePrefix
+      def get_path(item)
+        item.site_name.present? ? "/sites/#{item.site_name}#{super}" : super
+      end
+    end
+
     class Adapter < Office365::Adapter
       generate_id_helpers :share_point, 'share_point'
 
@@ -26,18 +32,24 @@ module ConnectorsSdk
       end
 
       class FileGraphItem < Office365::Adapter::FileGraphItem
+        include SitePrefix
+
         def self.convert_id_to_es_id(id)
           ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end
       end
 
       class FolderGraphItem < Office365::Adapter::FolderGraphItem
+        include SitePrefix
+
         def self.convert_id_to_es_id(id)
           ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end
       end
 
       class PackageGraphItem < Office365::Adapter::PackageGraphItem
+        include SitePrefix
+
         def self.convert_id_to_es_id(id)
           ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end

data/lib/connectors_sdk/share_point/{http_call_wrapper.rb → connector.rb}

@@ -6,22 +6,46 @@
 
 # frozen_string_literal: true
 
+require 'connectors_sdk/base/connector'
 require 'connectors_sdk/office365/config'
 require 'connectors_sdk/share_point/extractor'
 require 'connectors_sdk/share_point/authorization'
-require 'connectors_sdk/base/http_call_wrapper'
 
 module ConnectorsSdk
   module SharePoint
-    class HttpCallWrapper < ConnectorsSdk::Base::HttpCallWrapper
+    class Connector < ConnectorsSdk::Base::Connector
       SERVICE_TYPE = 'share_point'
 
-      def name
-        'SharePoint'
+      def compare_secrets(params)
+        missing_secrets?(params)
+
+        previous_user = client(:access_token => params[:other_secret][:access_token]).me
+        equivalent = previous_user.nil? ? false : previous_user.id == client(:access_token => params[:secret][:access_token]).me&.id
+
+        {
+          :equivalent => equivalent
+        }
+      end
+
+      def display_name
+        'SharePoint Online'
+      end
+
+      def connection_requires_redirect
+        true
       end
 
-      def service_type
-        SERVICE_TYPE
+      def configurable_fields
+        [
+          {
+            'key' => 'client_id',
+            'label' => 'Client ID'
+          },
+          {
+            'key' => 'client_secret',
+            'label' => 'Client Secret'
+          },
+        ]
       end
 
       private

data/lib/connectors_sdk/stub_connector/connector.rb

@@ -0,0 +1,62 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/connector'
+
+module ConnectorsSdk
+  module StubConnector
+    class Connector < ConnectorsSdk::Base::Connector
+      SERVICE_TYPE = 'stub_connector'
+
+      def display_name
+        'Stub Connector'
+      end
+
+      def configurable_fields
+        [
+          {
+            'key' => 'third_party_url',
+            'label' => 'Third Party URL'
+          },
+          {
+            'key' => 'third_party_api_key',
+            'label' => 'Third Party API Key'
+          }
+        ]
+      end
+
+      def health_check(_params)
+        true
+      end
+
+      def document_batch(_params)
+        results = 30.times.map do |i|
+          {
+            :action => :create_or_update,
+            :document => {
+              :id => "document_#{i}",
+              :type => 'document',
+              :body => "contents for document number: #{i}"
+            },
+            :download => nil
+          }
+        end
+
+        [results, {}, true]
+      end
+
+      def deleted(_params)
+        []
+      end
+
+      def permissions(_params)
+        []
+      end
+    end
+  end
+end

data/lib/connectors_shared/constants.rb

@@ -10,5 +10,17 @@ module ConnectorsShared
     SUBEXTRACTOR_RESERVED_FIELDS = %w[_subextracted_as_of _subextracted_version].freeze
     ALLOW_FIELD = '_allow_permissions'.freeze
     DENY_FIELD = '_deny_permissions'.freeze
+
+    # The following section reads as following:
+    # The job will extract documents until the job queue size will reach
+    # JOB_QUEUE_SIZE_IDLE_THRESHOLD items. After that, the job will attempt to sleep
+    # for IDLE_SLEEP_TIME seconds and check the queue size again. If the queue is still
+    # full, it will sleep for maximum MAX_IDDLE_ATTEMPTS times, and if the queue is still
+    # full, then job will be terminated.
+    JOB_QUEUE_SIZE_IDLE_THRESHOLD = 500 # How many documents the job queue stores until it sleeps
+    IDLE_SLEEP_TIME = 10 # For how long job queue will sleep before checking the queue size again
+    MAX_IDLE_ATTEMPTS = 30 # How many consecutive times job will try to sleep until it's destroyed
+
+    STALE_JOB_TIMEOUT = 60 * 30 # Time in seconds after which the job will be cleaned up if the job is considered stuck
   end
 end

data/lib/connectors_shared/job_status.rb

@@ -0,0 +1,18 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+module ConnectorsShared
+  class JobStatus
+    CREATED = 'created'
+    RUNNING = 'running'
+    FINISHED = 'finished'
+    FAILED = 'failed'
+
+    def self.is_valid?(status)
+      [CREATED, RUNNING, FINISHED, FAILED].include? status
+    end
+  end
+end

data/lib/connectors_shared/middleware/restrict_hostnames.rb

@@ -36,7 +36,7 @@ module ConnectorsShared
       def ips_from_hosts(hosts)
         hosts&.flat_map do |host|
           if URL_PATTERN.match(host)
-            lookup_ips(URI.parse(host).host)
+            lookup_ips(Addressable::URI.parse(host).hostname)
           elsif Resolv::IPv4::Regex.match(host) || Resolv::IPv6::Regex.match(host)
             IPAddr.new(host)
           else

data/lib/connectors_shared.rb

@@ -8,5 +8,6 @@ require 'connectors_shared/constants'
 require 'connectors_shared/errors'
 require 'connectors_shared/exception_tracking'
 require 'connectors_shared/extension_mapping_util'
+require 'connectors_shared/job_status'
 require 'connectors_shared/logger'
 require 'connectors_shared/monitor'