gitlab-cloud-connector 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c57fb113278c2e577aad185c79e45c5e640495c12a9bc0b48afbd925aa7c74a
-  data.tar.gz: 4639436cc983b23598ff7f46a58c87d502c36030f5a06becb339a97460a2889e
+  metadata.gz: 631ee0be48c3afe5732dcdd2cbf8f1f2bfb081ac40783a3790981072c80ab57d
+  data.tar.gz: b22b16aaf5e29d67081cd4ee03e83f572daeed2db9ab15962c0e82ed192676ab
 SHA512:
-  metadata.gz: 0030a33d736c6cfbd1f0af157a3627cc541ad3d379275c0649a78786881f12cefe06e316ca6d80bbf74a5cbdc34d728861312af8494efa376d85e3810b416765
-  data.tar.gz: d02c56fd8f8c79b3f64decc114be325c07bae61b7b15a6fbf741d1ed5de7b4cf2c7c9de9a616f3228a9202e1028e5fbe8cf3551ff83e92d9a44c490270ad689b
+  metadata.gz: c4cb3e67ea39fb9b7db24a5fe6b9375d8e1b8278c27bd91b929d11dd8bec180adbfb7a182d9f175a744ac3e7f2e490bd05bc888b7770d01b4b30dfe7f2aa3aff
+  data.tar.gz: 01ce43956fe3a3fc248757294520cf220dbc00b25b7d5f3774fc5320f467874f0f1ac1f43bbe91efcf07d1d9c80df7dbbfe21ce1ca429f3c5719a045396ebdd8

data/CONTRIBUTING.md

@@ -1,8 +1,8 @@
-## Making changes to this repository
+# Making changes to this repository
 
 We welcome contributions. When making changes:
 
-1. Follow good git commit practices. Summarize the change concisely in the commit title,
+1. Follow good Git commit practices. Summarize the change concisely in the commit title,
    and add relevant details in the commit body.
 1. Add a [changelog trailer](https://docs.gitlab.com/ee/user/project/changelogs.html) where it applies.
 1. Open a merge request.
@@ -44,7 +44,6 @@ This code of conduct applies both within project spaces and in public spaces
 when an individual is representing the project or its community.
 
 Instances of abusive, harassing, or otherwise unacceptable behavior can be
-reported by emailing contact@gitlab.com.
+reported by emailing <contact@gitlab.com>.
 
-This Code of Conduct is adapted from the [Contributor Covenant](https://contributor-covenant.org), version 1.1.0,
-available at [https://contributor-covenant.org/version/1/1/0/](https://contributor-covenant.org/version/1/1/0/).
+This Code of Conduct is adapted from the [Contributor Covenant, version 1.1.0](https://contributor-covenant.org/version/1/1/0/).

data/README.md

@@ -10,6 +10,54 @@ We expect Bundler is used to manage dependencies. To add the dependency, add it
 gem "gitlab-cloud-connector", "~> 0.1.1", require: 'cloud_connector'
 ```
 
+## Usage
+
+### Data Storage
+
+Cloud Connector uses YAML files stored in the project's `/config` directory. The data is organized by model type:
+
+```yaml
+# config/backend_services/ai_gateway.yml
+name: ai_gateway
+description: "AI Gateway"
+jwt_aud: gitlab-ai-gateway
+
+# config/add_ons/duo_pro.yml
+name: duo_pro
+description: "Duo Pro"
+
+# config/unit_primitives/duo_chat.yml
+name: duo_chat
+description: "Duo Chat"
+backend_services:
+  - ai_gateway
+add_ons:
+  - duo_pro
+  - duo_enterprise
+```
+
+### Working with DataModels
+
+`DataModels` are the core data structures in the Cloud Connector. Each model inherits from `DataModel::Base`. The data is loaded from YAML files into memory at runtime, making it efficient for read operations and relationship traversal.
+
+```ruby
+require 'cloud_connector'
+
+# Find unit primitive by name
+# Note:  value should be a symbol (e.g., `:duo_chat` instead of `'duo_chat'`).
+duo_chat = Gitlab::CloudConnector::DataModels::UnitPrimitive.find_by_name(:duo_chat)
+duo_chat.description
+duo_chat.cut_off_date
+
+# access associations
+duo_chat.add_ons
+duo_chat.license_types
+duo_chat.backend_services
+
+# read duo_chat audiences
+duo_chat.backend_services.map(&:jwt_aud)
+```
+
 ## Release Process
 
 See [Release Process](../../README.md#release-process)
@@ -18,4 +66,3 @@ Once the new gem version is visible on [RubyGems.org](https://rubygems.org/gems/
 it is recommended to update [GitLab's `Gemfile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/Gemfile)
 and [Customers Dot's `Gemfile`](https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/main/Gemfile) to bump the `gitlab-cloud-connector`
 Ruby gem to the new version also. See [Usage](#usage) for how to do this.
-

data/config/schemas/service_schema.json

@@ -0,0 +1,38 @@
+{
+  "$schema": "http://json-schema.org/draft-06/schema#",
+  "type": "object",
+  "description": "This is the legacy format for the service object that is being deprecated. Consider transitioning to the new format.",
+  "required": ["name"],
+  "properties": {
+    "name": {
+      "type": "string",
+      "pattern": "^[a-z0-9]+(_[a-z0-9]+)*$",
+      "description": "The unique name of the service, consisting of lowercase alphanumeric characters and underscores."
+    },
+    "basic_unit_primitive": {
+      "type": "string",
+      "pattern": "^[a-z0-9]+(_[a-z0-9]+)*$",
+      "description": "The basic unit primitive identifier."
+    },
+    "gitlab_realm": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "enum": ["gitlab-com", "self-managed"]
+      },
+      "description": "An array of unit primitives associated with the service."
+    },
+    "description": {
+      "type": "string",
+      "description": "A brief description of the service."
+    },
+    "unit_primitives": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "description": "An array of unit primitives associated with the service."
+    }
+  },
+  "additionalProperties": false
+}

data/config/services/agent_quick_actions.yml

@@ -0,0 +1,6 @@
+---
+name: agent_quick_actions
+gitlab_realm:
+  - self-managed
+unit_primitives:
+  - agent_quick_actions

data/config/services/anthropic_proxy.yml

@@ -0,0 +1,11 @@
+---
+name: anthropic_proxy
+unit_primitives:
+  - categorize_duo_chat_question
+  - generate_commit_message
+  - generate_issue_description
+  - glab_ask_git_command
+  - resolve_vulnerability
+  - review_merge_request
+  - summarize_issue_discussions
+  - summarize_review

data/config/services/code_suggestions.yml

@@ -0,0 +1,5 @@
+---
+name: code_suggestions
+unit_primitives:
+  - complete_code
+  - generate_code

data/config/services/duo_chat.yml

@@ -0,0 +1,21 @@
+---
+name: duo_chat
+basic_unit_primitive: duo_chat
+unit_primitives:
+  - ask_build
+  - ask_commit
+  - ask_epic
+  - ask_issue
+  - ask_merge_request
+  - documentation_search
+  - duo_chat
+  - explain_code
+  - fix_code
+  - include_dependency_context
+  - include_file_context
+  - include_issue_context
+  - include_local_git_context
+  - include_merge_request_context
+  - include_snippet_context
+  - refactor_code
+  - write_tests

data/config/services/duo_workflow.yml

@@ -0,0 +1,4 @@
+---
+name: duo_workflow
+unit_primitives:
+  - duo_workflow_execute_workflow

data/config/services/generate_description.yml

@@ -0,0 +1,4 @@
+---
+name: generate_description
+unit_primitives:
+  - generate_issue_description

data/config/services/sast.yml

@@ -0,0 +1,6 @@
+---
+name: sast
+gitlab_realm:
+  - gitlab-com
+unit_primitives:
+  - security_scans

data/config/services/self_hosted_models.yml

@@ -0,0 +1,24 @@
+---
+name: self_hosted_models
+gitlab_realm:
+  - gitlab-com
+unit_primitives:
+  - ask_build
+  - ask_commit
+  - ask_epic
+  - ask_issue
+  - ask_merge_request
+  - complete_code
+  - generate_code
+  - documentation_search
+  - duo_chat
+  - explain_code
+  - fix_code
+  - include_dependency_context
+  - include_file_context
+  - include_issue_context
+  - include_local_git_context
+  - include_merge_request_context
+  - include_snippet_context
+  - refactor_code
+  - write_tests

data/config/services/summarize_new_merge_request.yml

@@ -0,0 +1,4 @@
+---
+name: summarize_new_merge_request
+unit_primitives:
+  - summarize_merge_request

data/config/services/vertex_ai_proxy.yml

@@ -0,0 +1,9 @@
+---
+name: vertex_ai_proxy
+unit_primitives:
+  - generate_commit_message
+  - generate_cube_query
+  - glab_ask_git_command
+  - semantic_search_issue
+  - summarize_issue_discussions
+  - summarize_merge_request

data/config/unit_primitives/explain_code.yml

@@ -8,4 +8,5 @@ documentation_url: https://docs.gitlab.com/ee/user/gitlab_duo/#code-explanation
 backend_services:
   - ai_gateway
 add_ons:
+  - duo_pro
   - duo_enterprise

data/config/unit_primitives/generate_cube_query.yml

@@ -1,6 +1,7 @@
 ---
 name: generate_cube_query
 description: Convert plain text questions about event data in to a structured query in JSON format.
+cut_off_date: 2024-10-17T00:00:00+00:00
 group: group::platform insights
 feature_category: product_analytics_visualization
 backend_services:

data/config/unit_primitives/review_merge_request.yml

@@ -1,6 +1,7 @@
 ---
 name: review_merge_request
 description: Review new hunk and old hunk of a merge request diff.
+cut_off_date: 2024-10-17T00:00:00+00:00
 group: group::code review
 feature_category: code_review_workflow
 documentation_url: https://docs.gitlab.com/ee/user/gitlab_duo/#code-review

data/config/unit_primitives/summarize_merge_request.yml

@@ -1,6 +1,7 @@
 ---
 name: summarize_merge_request
 description: Summarize merge request from the comments.
+cut_off_date: 2024-10-17T00:00:00+00:00
 group: group::code review
 feature_category: code_review_workflow
 documentation_url: https://docs.gitlab.com/ee/user/gitlab_duo/#merge-request-summary

data/lib/cloud_connector.rb

@@ -5,6 +5,9 @@ require 'active_support/time'
 
 module Gitlab
   module CloudConnector
-    autoload(:JsonWebToken, 'gitlab/cloud_connector/json_web_token.rb')
+    autoload(:JsonWebToken, 'gitlab/cloud_connector/json_web_token')
+    autoload(:Configuration, 'gitlab/cloud_connector/configuration')
+    autoload(:DataModels, 'gitlab/cloud_connector/data_models')
+    autoload(:AvailableServicesGenerator, 'gitlab/cloud_connector/available_services_generator')
   end
 end

data/lib/gitlab/cloud_connector/available_services_generator.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/enumerable'
+require 'active_support/core_ext/object/blank'
+
+module Gitlab
+  module CloudConnector
+    class AvailableServicesGenerator
+      GITLAB_REALMS = %w[gitlab-com self-managed].freeze
+      WRONG_GITLAB_REALM_MESSAGE = 'Wrong gitlab_realm. Please use one of the following: %s'
+      IGNORE_UNIT_PRIMITIVES = %w[
+        agent_quick_actions
+        ask_build
+        ask_commit
+        ask_epic
+        ask_issue
+        ask_merge_request
+        categorize_duo_chat_question
+        complete_code
+        documentation_search
+        duo_workflow_execute_workflow
+        duo_workflow_generate_token
+        explain_code
+        fix_code
+        generate_code
+        generate_issue_description
+        refactor_code
+        security_scans
+        semantic_search_issue
+        summarize_issue_discussions
+        summarize_merge_request
+        write_tests
+      ].freeze
+
+      def generate(gitlab_realm)
+        raise WRONG_GITLAB_REALM_MESSAGE % GITLAB_REALMS.join(', ') unless GITLAB_REALMS.include?(gitlab_realm)
+
+        {
+          'services' => generate_services_config(gitlab_realm)
+        }
+      end
+
+      private
+
+      def generate_services_config(gitlab_realm)
+        services_config = {}
+
+        DataModels::Service.each do |service|
+          next if service.gitlab_realm && !service.gitlab_realm&.include?(gitlab_realm)
+
+          service_config = generate_service_config(service.unit_primitives, service.basic_unit_primitive)
+          # Remove duo_pro for self_hosted_models
+          service_config['bundled_with'].delete('duo_pro') if service.name == 'self_hosted_models'
+
+          services_config[service.name] = service_config
+        end
+
+        # Generate a stand-alone service for each unit primitive (except the ones in the ignore list)
+        unit_primitives = DataModels::UnitPrimitive.reject { |up| IGNORE_UNIT_PRIMITIVES.include?(up.name) }
+        unit_primitives.each do |unit_primitive|
+          next if services_config[unit_primitive.name]
+
+          services_config[unit_primitive.name] = generate_service_config([unit_primitive])
+        end
+
+        services_config.sort.to_h
+      end
+
+      def generate_service_config(unit_primitives, basic_unit_primitive = nil)
+        sample_primitive = unit_primitives.find { |up| up.name == basic_unit_primitive } if basic_unit_primitive
+        sample_primitive ||= unit_primitives.first
+        backend_service = sample_primitive&.backend_services&.first
+
+        {
+          'backend' => backend_service&.jwt_aud,
+          'cut_off_date' => sample_primitive&.cut_off_date&.strftime('%Y-%m-%d %H:%M:%S UTC'),
+          'min_gitlab_version' => sample_primitive&.min_gitlab_version,
+          'min_gitlab_version_for_free_access' => sample_primitive&.min_gitlab_version_for_free_access,
+          'bundled_with' => generate_bundled_config(unit_primitives),
+          'license_types' => sample_primitive&.license_types&.map(&:name)
+        }.compact_blank
+      end
+
+      def generate_bundled_config(unit_primitives)
+        unit_primitives.each_with_object({}) do |primitive, bundled|
+          target_groups = primitive.add_ons.any? ? primitive.add_ons.map(&:name) : ['_irrelevant']
+          target_groups.each do |group_name|
+            group_name = group_name.to_s
+            bundled[group_name] ||= { 'unit_primitives' => [] }
+            bundled[group_name]['unit_primitives'] << primitive.name
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/cloud_connector/configuration.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CloudConnector
+    module Configuration
+      class << self
+        attr_writer :config_dir
+
+        def config_dir
+          @config_dir ||= File.expand_path("../../../config", __dir__).freeze
+        end
+
+        def configure
+          yield self if block_given?
+        end
+
+        def reset!
+          @config_dir = nil
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/cloud_connector/data_model/associations.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CloudConnector
+    module DataModel
+      module Associations
+        module ClassMethods
+          # rubocop:disable Naming/PredicateName
+          def has_and_belongs_to_many(name)
+            define_method(name) do
+              instance_variable_get(:"@#{name}_association") ||
+                instance_variable_set(:"@#{name}_association", load_association_records(name))
+            end
+          end
+          # rubocop:enable Naming/PredicateName
+        end
+
+        def self.included(base)
+          base.extend ClassMethods
+        end
+
+        private
+
+        def load_association_records(association_name)
+          names = Array(self[association_name])
+          association_class = Gitlab::CloudConnector::DataModels.const_get(association_name.to_s.classify)
+          association_class.select { |record| names.include?(record.name.to_s) }
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/cloud_connector/data_model/base.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative 'associations'
+require_relative 'data_loader'
+
+module Gitlab
+  module CloudConnector
+    module DataModel
+      class Base
+        include Associations
+        extend Enumerable
+
+        class << self
+          def model_name
+            name&.demodulize
+          end
+
+          def each(&block)
+            all.each(&block)
+          end
+
+          def find_by_name(name)
+            all.find { |record| record.name.to_sym == name }
+          end
+
+          def all
+            @all ||= data_loader.load!
+          end
+
+          private
+
+          def data_loader
+            @data_loader ||= DataLoader.new(self)
+          end
+        end
+
+        def initialize(**opts)
+          opts.each do |key, value|
+            instance_variable_set(:"@#{key}", value)
+          end
+        end
+
+        def [](name)
+          instance_variable_get(:"@#{name}")
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/cloud_connector/data_model/data_loader.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+module Gitlab
+  module CloudConnector
+    module DataModel
+      class DataLoader
+        def initialize(model_class)
+          @model_class = model_class
+        end
+
+        def load!
+          Dir.glob(data_files_path).map { |file| load_model_from_file(file) }
+        end
+
+        private
+
+        attr_reader :model_class
+
+        def data_files_path
+          File.join(CloudConnector::Configuration.config_dir, model_class.model_name.tableize, '*.yml')
+        end
+
+        def load_model_from_file(file_path)
+          data = YAML.safe_load(
+            File.read(file_path),
+            permitted_classes: [Time],
+            symbolize_names: true
+          ).deep_symbolize_keys!
+
+          model_class.new(**data)
+        rescue StandardError => e
+          raise "Error loading file #{file_path}: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/cloud_connector/data_models.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative 'data_model/base'
+
+module Gitlab
+  module CloudConnector
+    module DataModels
+      class UnitPrimitive < DataModel::Base
+        has_and_belongs_to_many :backend_services
+        has_and_belongs_to_many :add_ons
+        has_and_belongs_to_many :license_types
+
+        attr_reader :cut_off_date, :deprecated_by_url, :deprecation_message, :description, :documentation_url,
+          :feature_category, :group, :introduced_by_url, :milestone, :min_gitlab_version,
+          :min_gitlab_version_for_free_access, :name, :unit_primitive_issue_url
+      end
+
+      class Service < DataModel::Base
+        has_and_belongs_to_many :unit_primitives
+
+        attr_reader :basic_unit_primitive, :description, :gitlab_realm, :name
+      end
+
+      class BackendService < DataModel::Base
+        attr_reader :description, :group, :jwt_aud, :name, :project_url
+      end
+
+      class LicenseType < DataModel::Base
+        attr_reader :description, :name
+      end
+
+      class AddOn < DataModel::Base
+        attr_reader :description, :name
+      end
+    end
+  end
+end

data/lib/gitlab/cloud_connector/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   module CloudConnector
-    VERSION = '0.2.2'
+    VERSION = '0.2.3'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-cloud-connector
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Nikola Milojevic
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-11-28 00:00:00.000000000 Z
+date: 2024-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -107,7 +107,18 @@ files:
 - config/schemas/add_on_schema.json
 - config/schemas/backend_service_schema.json
 - config/schemas/license_type_schema.json
+- config/schemas/service_schema.json
 - config/schemas/unit_primitive_schema.json
+- config/services/agent_quick_actions.yml
+- config/services/anthropic_proxy.yml
+- config/services/code_suggestions.yml
+- config/services/duo_chat.yml
+- config/services/duo_workflow.yml
+- config/services/generate_description.yml
+- config/services/sast.yml
+- config/services/self_hosted_models.yml
+- config/services/summarize_new_merge_request.yml
+- config/services/vertex_ai_proxy.yml
 - config/unit_primitives/agent_quick_actions.yml
 - config/unit_primitives/ask_build.yml
 - config/unit_primitives/ask_commit.yml
@@ -120,7 +131,6 @@ files:
 - config/unit_primitives/documentation_search.yml
 - config/unit_primitives/duo_chat.yml
 - config/unit_primitives/duo_workflow_execute_workflow.yml
-- config/unit_primitives/duo_workflow_generate_token.yml
 - config/unit_primitives/explain_code.yml
 - config/unit_primitives/explain_vulnerability.yml
 - config/unit_primitives/fix_code.yml
@@ -144,11 +154,16 @@ files:
 - config/unit_primitives/summarize_comments.yml
 - config/unit_primitives/summarize_issue_discussions.yml
 - config/unit_primitives/summarize_merge_request.yml
-- config/unit_primitives/summarize_new_merge_request.yml
 - config/unit_primitives/summarize_review.yml
 - config/unit_primitives/troubleshoot_job.yml
 - config/unit_primitives/write_tests.yml
 - lib/cloud_connector.rb
+- lib/gitlab/cloud_connector/available_services_generator.rb
+- lib/gitlab/cloud_connector/configuration.rb
+- lib/gitlab/cloud_connector/data_model/associations.rb
+- lib/gitlab/cloud_connector/data_model/base.rb
+- lib/gitlab/cloud_connector/data_model/data_loader.rb
+- lib/gitlab/cloud_connector/data_models.rb
 - lib/gitlab/cloud_connector/json_web_token.rb
 - lib/gitlab/cloud_connector/version.rb
 homepage: https://gitlab.com/gitlab-org/cloud-connector/gitlab-cloud-connector

data/config/unit_primitives/duo_workflow_generate_token.yml

@@ -1,10 +0,0 @@
----
-name: duo_workflow_generate_token
-description: Automate tasks and help increase productivity in your development workflow by using GitLab Duo Workflow.
-group: group::duo workflow
-feature_category: duo_workflow
-documentation_url: https://docs.gitlab.com/ee/user/gitlab_duo/#gitlab-duo-workflow
-introduced_by_url: https://gitlab.com/gitlab-org/customers-gitlab-com/-/merge_requests/10734
-unit_primitive_issue_url: https://gitlab.com/gitlab-org/duo-workflow/duo-workflow-service/-/issues/17
-backend_services:
-  - duo_workflow_service

data/config/unit_primitives/summarize_new_merge_request.yml

@@ -1,11 +0,0 @@
----
-name: summarize_new_merge_request
-description: Summarizes the changes for a new merge request based on the diff.
-cut_off_date: 2024-10-17T00:00:00+00:00 # this is the cut_off date used for duo_enterprise rollout (feature requires duo_enterprise add_on in order to be used).
-group: group::code review
-feature_category: code_review_workflow
-introduced_by_url: gitlab.com/gitlab-org/cloud-connector/gitlab-cloud-connector/-/merge_requests/54
-backend_services:
-  - ai_gateway
-add_ons:
-  - duo_enterprise