gitlab-triage 1.14.1 → 1.17.0

data/lib/gitlab/triage/engine.rb

@@ -33,6 +33,10 @@ module Gitlab
 
       DEFAULT_NETWORK_ADAPTER = Gitlab::Triage::NetworkAdapters::HttpartyAdapter
       DEFAULT_GRAPHQL_ADAPTER = Gitlab::Triage::NetworkAdapters::GraphqlAdapter
+      ALLOWED_STATE_VALUES = {
+        issues: %w[opened closed],
+        merge_requests: %w[opened closed merged]
+      }.with_indifferent_access.freeze
 
       def initialize(policies:, options:, network_adapter_class: DEFAULT_NETWORK_ADAPTER, graphql_network_adapter_class: DEFAULT_GRAPHQL_ADAPTER)
         options.host_url = policies.delete(:host_url) { options.host_url }
@@ -173,14 +177,23 @@ module Gitlab
         ExpandCondition.perform(rule_conditions(rule)) do |conditions|
           # retrieving the resources for every rule is inefficient
           # however, previous rules may affect those upcoming
-          resources = network.query_api(build_get_url(resource_type, conditions))
-          iids = resources.pluck('iid').map(&:to_s)
+          resources = []
+
+          if rule[:api] == 'graphql'
+            graphql_query = build_graphql_query(resource_type, conditions, true)
+            resources = graphql_network.query(graphql_query, source: source_full_path)
+          else
+            resources = network.query_api(build_get_url(resource_type, conditions))
+            iids = resources.pluck('iid').map(&:to_s)
+
+            graphql_query = build_graphql_query(resource_type, conditions)
+            graphql_resources = graphql_network.query(graphql_query, source: source_full_path, iids: iids) if graphql_query.any?
+
+            decorate_resources_with_graphql_data(resources, graphql_resources)
+          end
 
-          graphql_query = build_graphql_query(resource_type, conditions)
-          graphql_resources = graphql_network.query(graphql_query, source: options.source_id, iids: iids) if graphql_query.present?
           # In some filters/actions we want to know which resource type it is
           attach_resource_type(resources, resource_type)
-          decorate_resources_with_graphql_data(resources, graphql_resources)
 
           puts "\n\n* Found #{resources.count} resources..."
           print "* Filtering resources..."
@@ -195,10 +208,17 @@ module Gitlab
         end
       end
 
-      # We don't have to do this once the response will contain the type
-      # of the resource. For now let's just attach it.
       def attach_resource_type(resources, resource_type)
-        resources.each { |resource| resource[:type] ||= resource_type }
+        resources.each { |resource| resource[:type] = resource_type }
+        # TODO:                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        # We should not overwrite the attribute here, but we need to
+        # fix it first. We should instead use something like
+        # gitlab_triage_resource_type so it won't conflict with the
+        # existing fields.
+        # And we need to retain the backward compatibility that using
+        # {{type}} will give us this value, rather than from the REST API,
+        # which will give us ISSUE from:
+        # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/59648
       end
 
       def decorate_resources_with_graphql_data(resources, graphql_resources)
@@ -276,7 +296,13 @@ module Gitlab
           condition_builders << APIQueryBuilders::MultiQueryParamBuilder.new('not[labels]', conditions[:forbidden_labels], ',')
         end
 
-        condition_builders << APIQueryBuilders::SingleQueryParamBuilder.new('state', conditions[:state]) if conditions[:state]
+        if conditions[:state]
+          condition_builders << APIQueryBuilders::SingleQueryParamBuilder.new(
+            'state',
+            conditions[:state],
+            allowed_values: ALLOWED_STATE_VALUES[resource_type])
+        end
+
         condition_builders << APIQueryBuilders::SingleQueryParamBuilder.new('milestone', Array(conditions[:milestone])[0]) if conditions[:milestone]
         condition_builders << APIQueryBuilders::SingleQueryParamBuilder.new('source_branch', conditions[:source_branch]) if conditions[:source_branch]
         condition_builders << APIQueryBuilders::SingleQueryParamBuilder.new('target_branch', conditions[:target_branch]) if conditions[:target_branch]
@@ -303,9 +329,24 @@ module Gitlab
         ).build
       end
 
-      def build_graphql_query(resource_type, conditions)
+      def build_graphql_query(resource_type, conditions, graphql_only = false)
         Gitlab::Triage::GraphqlQueries::QueryBuilder
-          .new(options.source, resource_type, conditions)
+          .new(options.source, resource_type, conditions, graphql_only: graphql_only)
+      end
+
+      def source_full_path
+        @source_full_path ||= fetch_source_full_path
+      end
+
+      def fetch_source_full_path
+        return options.source_id unless /\A\d+\z/.match?(options.source_id)
+
+        source_details = network.query_api(build_get_url(nil, {})).first
+        full_path = source_details['full_path'] || source_details['path_with_namespace']
+
+        raise ArgumentError, 'A source with given source_id was not found!' if full_path.blank?
+
+        full_path
       end
     end
   end

data/lib/gitlab/triage/errors/network.rb

@@ -3,6 +3,8 @@ module Gitlab
     module Errors
       module Network
         InternalServerError = Class.new(StandardError)
+        TooManyRequests = Class.new(StandardError)
+        UnexpectedResponse = Class.new(StandardError)
       end
     end
   end

data/lib/gitlab/triage/filters/discussions_conditions_filter.rb

@@ -36,9 +36,7 @@ module Gitlab
           if @attribute == :notes
             @resource[:user_notes_count]
           else
-            @resource.dig(:discussions, :nodes)&.count do |node|
-              !node&.dig(:notes, :nodes, 0, :system)
-            end
+            @resource[:user_discussions_count]
           end
         end
 

data/lib/gitlab/triage/graphql_network.rb

@@ -11,6 +11,8 @@ module Gitlab
     class GraphqlNetwork
       attr_reader :options, :adapter
 
+      MINIMUM_RATE_LIMIT = 25
+
       def initialize(adapter)
         @adapter = adapter
         @options = adapter.options
@@ -33,22 +35,47 @@ module Gitlab
             variables: variables.merge(after: response.delete(:end_cursor))
           )
 
+          rate_limit_debug(response) if options.debug
+          rate_limit_wait(response)
+
           resources.concat(Array.wrap(response.delete(:results)))
         end while response.delete(:more_pages)
 
         resources
           .map { |resource| resource.deep_transform_keys(&:underscore) }
           .map(&:with_indifferent_access)
-          .map { |resource| resource.merge(id: extract_id_from_global_id(resource[:id])) }
+          .map { |resource| normalize(resource) }
       end
 
       private
 
+      def normalize(resource)
+        resource
+          .slice(:iid, :state, :author, :merged_at, :user_notes_count, :user_discussions_count, :upvotes, :downvotes, :project_id, :web_url)
+          .merge(
+            id: extract_id_from_global_id(resource[:id]),
+            labels: [*resource.dig(:labels, :nodes)].pluck(:title),
+            assignees: [*resource.dig(:assignees, :nodes)]
+          )
+      end
+
       def extract_id_from_global_id(global_id)
         return if global_id.blank?
 
         GlobalID.parse(global_id).model_id.to_i
       end
+
+      def rate_limit_debug(response)
+        rate_limit_infos = "Rate limit remaining: #{response[:ratelimit_remaining]} (reset at #{response[:ratelimit_reset_at]})"
+        puts Gitlab::Triage::UI.debug "rate_limit_infos: #{rate_limit_infos}"
+      end
+
+      def rate_limit_wait(response)
+        return unless response.delete(:ratelimit_remaining) < MINIMUM_RATE_LIMIT
+
+        puts Gitlab::Triage::UI.debug "Rate limit almost exceeded, sleeping for #{response[:ratelimit_reset_at] - Time.now} seconds" if options.debug
+        sleep(1) until Time.now >= response[:ratelimit_reset_at]
+      end
     end
   end
 end

data/lib/gitlab/triage/graphql_queries/query_builder.rb

@@ -1,14 +1,16 @@
-require_relative 'user_notes_query'
-require_relative 'threads_query'
+require_relative 'query_param_builders/base_param_builder'
+require_relative 'query_param_builders/date_param_builder'
+require_relative 'query_param_builders/labels_param_builder'
 
 module Gitlab
   module Triage
     module GraphqlQueries
       class QueryBuilder
-        def initialize(source_type, resource_type, conditions)
+        def initialize(source_type, resource_type, conditions, graphql_only: false)
           @source_type = source_type.to_s.singularize
           @resource_type = resource_type
           @conditions = conditions
+          @graphql_only = graphql_only
         end
 
         def resource_path
@@ -16,30 +18,84 @@ module Gitlab
         end
 
         def query
-          return if query_template.nil?
+          return if resource_fields.empty?
 
-          format(query_template, source_type: source_type, resource_type: resource_type.to_s.camelize(:lower), group_query: group_query)
+          format(
+            BASE_QUERY,
+            source_type: source_type,
+            resource_type: resource_type.to_s.camelize(:lower),
+            resource_fields: resource_fields.join(' '),
+            resource_query: resource_query,
+            iids_declaration: graphql_only ? nil : ', $iids: [String!]',
+            iids_query: graphql_only ? nil : ', iids: $iids'
+          )
         end
 
-        delegate :present?, to: :query_template
+        delegate :any?, to: :resource_fields
 
         private
 
-        attr_reader :source_type, :resource_type, :conditions
+        attr_reader :source_type, :resource_type, :conditions, :graphql_only
 
-        def query_template
-          case conditions.dig(:discussions, :attribute).to_s
-          when 'notes'
-            UserNotesQuery
-          when 'threads'
-            ThreadsQuery
+        BASE_QUERY = <<~GRAPHQL.freeze
+          query($source: ID!, $after: String%{iids_declaration}) {
+            %{source_type}(fullPath: $source) {
+              id
+              %{resource_type}(after: $after%{iids_query}%{resource_query}) {
+                pageInfo {
+                  hasNextPage
+                  endCursor
+                }
+                nodes {
+                  id iid title updatedAt createdAt webUrl projectId %{resource_fields}
+                }
+              }
+            }
+          }
+        GRAPHQL
+
+        def resource_fields
+          fields = []
+
+          fields << 'userNotesCount' if conditions.dig(:discussions, :attribute).to_s == 'notes'
+          fields << 'userDiscussionsCount' if conditions.dig(:discussions, :attribute).to_s == 'threads'
+
+          if graphql_only
+            fields << 'labels { nodes { title } }'
+            fields << 'author { id name username }'
+            fields << 'assignees { nodes { id name username } }' if conditions.key?(:assignee_member)
+            fields << 'upvotes' if conditions.dig(:upvotes, :attribute).to_s == 'upvotes'
+            fields << 'downvotes' if conditions.dig(:upvotes, :attribute).to_s == 'downvotes'
+            fields << 'mergedAt' if resource_type == 'merge_requests'
           end
+
+          fields
         end
 
-        def group_query
-          return if source_type != 'group'
+        def resource_query
+          condition_queries = []
+
+          condition_queries << QueryParamBuilders::BaseParamBuilder.new('includeSubgroups', true, with_quotes: false) if source_type == 'group'
+
+          conditions.each do |condition, condition_params|
+            condition_queries << QueryParamBuilders::DateParamBuilder.new(condition_params) if condition.to_s == 'date'
+            condition_queries << QueryParamBuilders::BaseParamBuilder.new('milestoneTitle', condition_params) if condition.to_s == 'milestone'
+            condition_queries << QueryParamBuilders::BaseParamBuilder.new('state', condition_params, with_quotes: false) if condition.to_s == 'state'
+
+            if resource_type == 'merge_requests'
+              condition_queries << QueryParamBuilders::LabelsParamBuilder.new('labels', condition_params) if condition.to_s == 'labels'
+              condition_queries << QueryParamBuilders::BaseParamBuilder.new('sourceBranch', condition_params) if condition.to_s == 'source_branch'
+              condition_queries << QueryParamBuilders::BaseParamBuilder.new('targetBranch', condition_params) if condition.to_s == 'target_branch'
+            end
+
+            if resource_type == 'issues'
+              condition_queries << QueryParamBuilders::LabelsParamBuilder.new('labelName', condition_params) if condition.to_s == 'labels'
+            end
+          end
 
-          ', includeSubgroups: true'
+          condition_queries
+            .map(&:build_param)
+            .join
         end
       end
     end

data/lib/gitlab/triage/graphql_queries/query_param_builders/base_param_builder.rb

@@ -0,0 +1,25 @@
+require_relative '../../utils'
+
+module Gitlab
+  module Triage
+    module GraphqlQueries
+      module QueryParamBuilders
+        class BaseParamBuilder
+          attr_reader :param_name, :param_contents, :with_quotes
+
+          def initialize(param_name, param_contents, with_quotes: true)
+            @param_name = param_name
+            @param_contents = param_contents.to_s.strip
+            @with_quotes = with_quotes
+          end
+
+          def build_param
+            contents = with_quotes ? Utils.graphql_quote(param_contents) : param_contents
+
+            ", #{param_name}: #{contents}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/triage/graphql_queries/query_param_builders/date_param_builder.rb

@@ -0,0 +1,35 @@
+require_relative '../../param_builders/date_param_builder'
+require_relative 'base_param_builder'
+
+module Gitlab
+  module Triage
+    module GraphqlQueries
+      module QueryParamBuilders
+        class DateParamBuilder < BaseParamBuilder
+          ATTRIBUTES = %w[updated_at created_at merged_at].freeze
+
+          def initialize(condition_hash)
+            date_param_builder = ParamBuilders::DateParamBuilder.new(ATTRIBUTES, condition_hash)
+
+            super(build_param_name(condition_hash), date_param_builder.param_content)
+          end
+
+          private
+
+          def build_param_name(condition_hash)
+            prefix = condition_hash[:attribute].to_s.sub(/_at\z/, '')
+            suffix =
+              case condition_hash[:condition].to_sym
+              when :older_than
+                'Before'
+              when :newer_than
+                'After'
+              end
+
+            "#{prefix}#{suffix}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/triage/graphql_queries/query_param_builders/labels_param_builder.rb

@@ -0,0 +1,18 @@
+require_relative '../../utils'
+require_relative 'base_param_builder'
+
+module Gitlab
+  module Triage
+    module GraphqlQueries
+      module QueryParamBuilders
+        class LabelsParamBuilder < BaseParamBuilder
+          def initialize(param_name, labels)
+            label_param_content = labels.map { |label| Utils.graphql_quote(label) }.join(', ').then { |content| "[#{content}]" }
+
+            super(param_name, label_param_content, with_quotes: false)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/triage/network.rb

@@ -33,7 +33,9 @@ module Gitlab
           print '.'
           url = response.fetch(:next_page_url) { url }
 
-          response = execute_with_retry([Net::ReadTimeout, Errors::Network::InternalServerError]) do
+          response = execute_with_retry(
+            exception_types: [Net::ReadTimeout, Errors::Network::InternalServerError],
+            backoff_exceptions: Errors::Network::TooManyRequests) do
             puts Gitlab::Triage::UI.debug "query_api: #{url}" if options.debug
 
             @adapter.get(token, url)
@@ -44,8 +46,10 @@ module Gitlab
           case results
           when Array
             resources.concat(results)
-          else
+          when Hash
             resources << results
+          else
+            raise Errors::Network::UnexpectedResponse, "Unexpected response: #{results.inspect}"
           end
 
           rate_limit_debug(response) if options.debug
@@ -58,7 +62,9 @@ module Gitlab
       end
 
       def post_api(url, body)
-        response = execute_with_retry(Net::ReadTimeout) do
+        response = execute_with_retry(
+          exception_types: Net::ReadTimeout,
+          backoff_exceptions: Errors::Network::TooManyRequests) do
           puts Gitlab::Triage::UI.debug "post_api: #{url}" if options.debug
 
           @adapter.post(token, url, body)

data/lib/gitlab/triage/network_adapters/graphql_adapter.rb

@@ -1,3 +1,4 @@
+require 'httparty'
 require 'graphql/client'
 require 'graphql/client/http'
 
@@ -17,16 +18,22 @@ module Gitlab
           raise_on_error!(response)
 
           parsed_response = parse_response(response, resource_path)
+          headers = response.extensions.fetch('headers', {})
 
-          return { results: {} } if parsed_response.nil?
-          return { results: parsed_response.map(&:to_h) } if parsed_response.is_a?(Client::List)
-          return { results: parsed_response.to_h } unless parsed_response.nodes?
+          graphql_response = {
+            ratelimit_remaining: headers['ratelimit-remaining'].to_i,
+            ratelimit_reset_at: Time.at(headers['ratelimit-reset'].to_i)
+          }
+
+          return graphql_response.merge(results: {}) if parsed_response.nil?
+          return graphql_response.merge(results: parsed_response.map(&:to_h)) if parsed_response.is_a?(Client::List)
+          return graphql_response.merge(results: parsed_response.to_h) unless parsed_response.nodes?
 
-          {
+          graphql_response.merge(
             more_pages: parsed_response.page_info.has_next_page,
             end_cursor: parsed_response.page_info.end_cursor,
             results: parsed_response.nodes.map(&:to_h)
-          }
+          )
         end
 
         delegate :parse, to: :client
@@ -47,11 +54,27 @@ module Gitlab
 
         def http_client
           Client::HTTP.new("#{options.host_url}/api/graphql") do
-            def headers(context) # rubocop:disable Lint/NestedMethodDefinition
-              {
-                'Content-type' => 'application/json',
-                'PRIVATE-TOKEN' => context[:token]
-              }
+            def execute(document:, operation_name: nil, variables: {}, context: {}) # rubocop:disable Lint/NestedMethodDefinition
+              body = {}
+              body['query'] = document.to_query_string
+              body['variables'] = variables if variables.any?
+              body['operationName'] = operation_name if operation_name
+
+              response = HTTParty.post(
+                uri,
+                body: body.to_json,
+                headers: {
+                  'Content-type' => 'application/json',
+                  'PRIVATE-TOKEN' => context[:token]
+                }
+              )
+
+              case response.code
+              when 200, 400
+                JSON.parse(response.body).merge('extensions' => { 'headers' => response.headers })
+              else
+                { 'errors' => [{ 'message' => "#{response.code} #{response.message}" }] }
+              end
             end
           end
         end