gitlab-labkit 0.37.0 → 0.40.0

data/README.md

@@ -1,4 +1,4 @@
-# LabKit-Ruby 🔬🔬🔬🔬🔬
+# LabKit-Ruby 🔬🔬🔬🔬🔬
 
 LabKit-Ruby is minimalist library to provide functionality for Ruby services at GitLab.
 
@@ -19,9 +19,12 @@ The changelog is available via [**tagged release notes**](https://gitlab.com/git
 LabKit-Ruby provides functionality in a number of areas:
 
 1. `Labkit::Context` used for providing context information to log messages.
-1. `Labkit::Correlation` For accessing the correlation id. (Generated and propagated by `Labkit::Context`)
+1. `Labkit::Correlation` for accessing the correlation id. (Generated and propagated by `Labkit::Context`)
+1. `Labkit::CoveredExperience` for tracking covered experiences. More on the [README](./lib/labkit/covered_experiences/README.md).
 1. `Labkit::FIPS` for checking for FIPS mode and using FIPS-compliant algorithms.
 1. `Labkit::Logging` for sanitizing log messages.
+1. `Labkit::Metrics` for metrics. More on the [README](./lib/labkit/metrics/README.md).
+1. `Labkit::RSpec` for RSpec matchers to test Labkit functionality (requires selective loading). More on the [README](./lib/labkit/rspec/README.md).
 1. `Labkit::Tracing` for handling and propagating distributed traces.
 
 ## Developing
@@ -40,8 +43,6 @@ $ # Run tests, linters
 $ bundle exec rake verify
 ```
 
-Note that LabKit-Ruby uses the [`rufo`](https://github.com/ruby-formatter/rufo) for auto-formatting. Please run `bundle exec rake fix` to auto-format your code before pushing.
-
 Please also review the [development section of the LabKit (go) README](https://gitlab.com/gitlab-org/labkit#developing-labkit) for details of the LabKit architectural philosophy.
 
 To work on some of the scripts we use for releasing a new version,
@@ -51,7 +52,7 @@ make sure to add a new `.env.sh`.
 cp .env.example.sh .env.sh`
 ```
 
-Inside `.env.sh`, add a personal acccess token for the `GITLAB_TOKEN`
+Inside `.env.sh`, add a personal acccess token for the `CHANGELOG_GITLAB_TOKEN`
 environment variable. Next source the file:
 
 ```console

data/Rakefile

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "bundler/gem_tasks"
-require "rufo"
 
 require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
@@ -11,27 +10,9 @@ RuboCop::RakeTask.new(:rubocop) do |task|
   task.options = %w[--parallel]
 end
 
-desc "Alias for `rake rufo:run`"
-task :format => ["rufo:run"]
-
-namespace :rufo do
-  require "rufo"
-
-  def rufo_command(*switches, rake_args)
-    files_or_dirs = rake_args[:files_or_dirs] || "."
-    args = switches + files_or_dirs.split(" ")
-    Rufo::Command.run(args)
-  end
-
-  desc "Format Ruby code in current directory"
-  task :run, [:files_or_dirs] do |_task, rake_args|
-    rufo_command(rake_args)
-  end
-
-  desc "Check that no formatting changes are produced"
-  task :check, [:files_or_dirs] do |_task, rake_args|
-    rufo_command("--check", rake_args)
-  end
+desc "update the rubocop todo config"
+RuboCop::RakeTask.new("rubocop:config") do |task|
+  task.options = %w[--auto-gen-config]
 end
 
 desc "Generate test protobuf stubs"
@@ -40,8 +21,17 @@ task :gen_test_proto do
   Rufo::Command.run(["spec/support/grpc_service/test_pb.rb", "spec/support/grpc_service/test_services_pb.rb"])
 end
 
-task :fix => %w[rufo:run rubocop:auto_correct]
+task :fix => ["rubocop:autocorrect"]
 
-task :verify => %w[spec rufo:check rubocop]
+task :verify => %w[spec rubocop]
 
 task :default => %w[verify build]
+
+desc "Start an IRB console with gem pre-loaded"
+task :console do
+  $LOAD_PATH.unshift(File.expand_path("lib", __dir__))
+  require "irb"
+  require "gitlab-labkit"
+  ARGV.clear
+  IRB.start
+end

data/config/covered_experiences/schema.json

@@ -0,0 +1,35 @@
+{
+  "$schema": "http://json-schema.org/draft-06/schema#",
+  "$id": "https://gitlab.com/gitlab-org/ruby/gems/labkit-ruby/-/raw/master/config/covered_experiences/schema.json",
+  "title": "Covered Experience Definition",
+  "description": "Schema for GitLab Covered Experience files",
+  "type": "object",
+  "properties": {
+    "description": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Human-readable description of the covered experience"
+    },
+    "feature_category": {
+      "type": "string",
+      "minLength": 1,
+      "description": "GitLab feature category this experience belongs to"
+    },
+    "urgency": {
+      "type": "string",
+      "enum": [
+        "sync_fast",
+        "sync_slow",
+        "async_fast",
+        "async_slow"
+      ],
+      "description": "Urgency level for this covered experience"
+    }
+  },
+  "required": [
+    "description",
+    "feature_category",
+    "urgency"
+  ]
+}
+

data/config/covered_experiences/testing_sample.yml

@@ -0,0 +1,4 @@
+# yaml-language-server: $schema=./schema.json
+description: "Creating a new merge request in a project"
+feature_category: "source_code_management"
+urgency: "sync_fast"

data/gitlab-labkit.gemspec

@@ -5,7 +5,8 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name = "gitlab-labkit"
-  spec.version = `git describe --tags`.chomp.gsub(/^v/, "")
+  version = ENV["CI_COMMIT_TAG"] || `git describe --tag --match "v[0-9]*.[0-9]*.[0-9]*"`
+  spec.version = version.chomp.gsub(/^v/, "")
   spec.authors = ["Andrew Newdigate"]
   spec.email = ["andrew@gitlab.com"]
 
@@ -16,32 +17,36 @@ Gem::Specification.new do |spec|
 
   spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|tools)/}) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = ">= 2.6.0"
+  spec.required_ruby_version = "~> 3.2"
 
   # Please maintain alphabetical order for dependencies
   spec.add_runtime_dependency "actionpack", ">= 5.0.0", "< 8.1.0"
   spec.add_runtime_dependency "activesupport", ">= 5.0.0", "< 8.1.0"
   spec.add_runtime_dependency "grpc", ">= 1.62" # Be sure to update the "grpc-tools" dev_dependency too
+  spec.add_runtime_dependency "google-protobuf", "~> 3" # Keep the major version to 3 until we update the `grpc` gem
   spec.add_runtime_dependency "jaeger-client", "~> 1.1.0"
+  spec.add_runtime_dependency 'json-schema', '~> 5.1'
   spec.add_runtime_dependency "opentracing", "~> 0.4"
-  spec.add_runtime_dependency "pg_query", ">= 5.1.0", "< 7.0"
+  spec.add_runtime_dependency "pg_query", ">= 6.1.0", "< 7.0"
+  spec.add_runtime_dependency "prometheus-client-mmap", "~> 1.2.9"
   spec.add_runtime_dependency "redis", "> 3.0.0", "< 6.0.0"
 
   # Please maintain alphabetical order for dev dependencies
   spec.add_development_dependency "excon", "~> 0.78.1"
   spec.add_development_dependency "faraday", "~> 1.10.3"
   spec.add_development_dependency "gitlab-dangerfiles", "~> 2.11.0"
-  spec.add_development_dependency "gitlab-styles", "~> 6.2.0"
+  spec.add_development_dependency "gitlab-styles", "~> 13.0.2"
   spec.add_development_dependency "grpc-tools", ">= 1.62"
-  spec.add_development_dependency "httparty", "~> 0.17.3"
-  spec.add_development_dependency "httpclient", "~> 2.8.3"
+  spec.add_development_dependency "httparty", "~> 0.22.0"
+  spec.add_development_dependency "httpclient", "~> 2.9.0"
+  spec.add_development_dependency "irb", "~> 1.15.2"
   spec.add_development_dependency "pry", "~> 0.12"
+  spec.add_development_dependency "pry-byebug", "~> 3.11"
   spec.add_development_dependency "rack", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 12.3"
+  spec.add_development_dependency "rake", "~> 13.2"
   spec.add_development_dependency "rest-client", "~> 2.1.0"
   spec.add_development_dependency "rspec", "~> 3.12.0"
   spec.add_development_dependency "rspec-parameterized", "~> 1.0"
-  spec.add_development_dependency "rufo", "0.9.0"
   spec.add_development_dependency "sidekiq", ">= 5.2", "< 7"
   spec.add_development_dependency "webrick", "~> 1.7.0"
 end

data/lib/gitlab-labkit.rb

@@ -7,11 +7,13 @@
 module Labkit
   autoload :System, "labkit/system"
 
-  autoload :Correlation, "labkit/correlation"
   autoload :Context, "labkit/context"
+  autoload :Correlation, "labkit/correlation"
+  autoload :CoveredExperience, "labkit/covered_experience"
   autoload :FIPS, "labkit/fips"
   autoload :Tracing, "labkit/tracing"
   autoload :Logging, "labkit/logging"
+  autoload :Metrics, "labkit/metrics"
   autoload :Middleware, "labkit/middleware"
 
   # Publishers to publish notifications whenever a HTTP reqeust is made.

data/lib/labkit/context.rb

@@ -5,6 +5,7 @@ require "securerandom"
 require "active_support/core_ext/module/delegation"
 require "active_support/core_ext/string/starts_ends_with"
 require "active_support/core_ext/string/inflections"
+require "active_support/core_ext/object/blank"
 
 module Labkit
   # A context can be used to provide structured information on what resources

data/lib/labkit/covered_experience/README.md

@@ -0,0 +1,134 @@
+# Covered Experience
+
+This module covers the definition for Covered Experiences, as described in the [blueprint](https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/covered_experience_slis/#covered-experience-definition).
+
+## Configuration
+
+### Logger Configuration
+
+By default, `Labkit::CoveredExperience` uses `Labkit::Logging::JsonLogger.new($stdout)` for logging. You can configure a custom logger:
+
+```ruby
+Labkit::CoveredExperience.configure do |config|
+  config.logger = Labkit::Logging::JsonLogger.new($stdout)
+end
+```
+
+This configuration affects all Covered Experience instances and their logging output.
+
+### Covered Experience Definitions
+
+Covered experience definitions will be lazy loaded from the default directory (`config/covered_experiences`).
+
+Create a new covered experience file in the registry directory, e.g. config/covered_experiences/merge_request_creation.yaml
+
+The basename of the file will be taken as the covered_experience_id.
+
+The schema header is optional, but if you're using VSCode (or any other editor with support), you can get them validated
+instantaneously in the editor via a [JSON schema plugin](https://marketplace.visualstudio.com/items?itemName=remcohaszing.schemastore).
+
+```yaml
+# yaml-language-server: $schema=https://gitlab.com/gitlab-org/ruby/gems/labkit-ruby/-/raw/master/config/covered_experiences/schema.json
+description: "Creating a new merge request in a project"
+feature_category: "source_code_management"
+urgency: "sync_fast"
+```
+
+**Feature category**
+
+https://docs.gitlab.com/development/feature_categorization/#feature-categorization.
+
+**Urgency**
+
+| Threshold    | Description                                                                                                                                                      | Examples                                                                       | Value |
+|--------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------|-------|
+| `sync_fast`  | A user is awaiting a synchronous response which needs to be returned before they can continue with their action                                                  | A full-page render                                                             | 2s    |
+| `sync_slow`  | A user is awaiting a synchronous response which needs to be returned before they can continue with their action, but which the user may accept a slower response | Displaying a full-text search response while displaying an amusement animation | 5s    |
+| `async_fast` | An async process which may block a user from continuing with their user journey                                                                                  | MR diff update after git push                                                  | 15s   |
+| `async_slow` | An async process which will not block a user and will not be immediately noticed as being slow                                                                   | Notification following an assignment                                           | 5m    |
+
+## Usage
+
+The `Labkit::CoveredExperience` module provides a simple API for measuring and tracking covered experiences in your application.
+
+
+#### Accessing a Covered Experience
+
+```ruby
+# Get a covered experience by ID
+experience = Labkit::CoveredExperience.get('merge_request_creation')
+```
+
+#### Using with a Block (Recommended)
+
+The simplest way to use covered experiences is with a block, which automatically handles starting and completing the experience:
+
+```ruby
+Labkit::CoveredExperience.start('merge_request_creation') do |experience|
+  # Your code here
+  create_merge_request
+
+  # Add checkpoints for important milestones
+  experience.checkpoint
+
+  validate_merge_request
+  experience.checkpoint
+
+  send_notifications
+end
+```
+
+#### Manual Control
+
+For more control, you can manually start, checkpoint, and complete experiences:
+
+```ruby
+experience = Labkit::CoveredExperience.get('merge_request_creation')
+experience.start
+
+# Perform some work
+create_merge_request
+
+# Mark important milestones
+experience.checkpoint
+
+# Perform more work
+validate_merge_request
+experience.checkpoint
+
+# Complete the experience
+experience.complete
+```
+
+### Error Handling
+
+When using the block form, errors are automatically captured:
+
+```ruby
+Labkit::CoveredExperience.start('merge_request_creation') do |experience|
+  # If this raises an exception, it will be captured automatically
+  risky_operation
+end
+```
+
+For manual control, you can mark errors explicitly:
+
+```ruby
+experience = Labkit::CoveredExperience.get('merge_request_creation')
+experience.start
+
+begin
+  risky_operation
+rescue StandardError => e
+  experience.error!(e)
+  raise
+ensure
+  experience.complete
+end
+```
+
+### Error Behavior
+
+- In `development` and `test` environments, accessing a non-existent covered experience will raise a `NotFoundError`
+- In other environments, a null object is returned that safely ignores all method calls
+- Attempting to checkpoint or complete an unstarted experience will raise an `UnstartedError` in `development` and `test` environments

data/lib/labkit/covered_experience/error.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Labkit
+  module CoveredExperience
+    CoveredExperienceError = Class.new(StandardError)
+    UnstartedError = Class.new(CoveredExperienceError)
+    NotFoundError = Class.new(CoveredExperienceError)
+  end
+end

data/lib/labkit/covered_experience/experience.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require 'labkit/context'
+require 'labkit/covered_experience/error'
+
+module Labkit
+  module CoveredExperience
+    URGENCY_THRESHOLDS_IN_SECONDS = {
+      sync_fast: 2,
+      sync_slow: 5,
+      async_fast: 15,
+      async_slow: 300
+    }.freeze
+
+    # The `Experience` class represents a single Covered Experience
+    # event to be measured and reported.
+    class Experience
+      attr_reader :error
+
+      def initialize(definition)
+        @definition = definition
+      end
+
+      # Start the Covered Experience.
+      #
+      # @yield [self] When a block is provided, the experience will be completed automatically.
+      # @param extra [Hash] Additional data to include in the log event
+      # @return [self]
+      # @raise [CoveredExperienceError] If the block raises an error.
+      #
+      # Usage:
+      #
+      #  CoveredExperience.new(definition).start do |experience|
+      #    experience.checkpoint
+      #    experience.checkpoint
+      #  end
+      #
+      #  experience = CoveredExperience.new(definition)
+      #  experience.start
+      #  experience.checkpoint
+      #  experience.complete
+      def start(**extra, &)
+        @start_time = Time.now.utc
+        checkpoint_counter.increment(checkpoint: "start")
+        log_event("start", **extra)
+
+        return self unless block_given?
+
+        begin
+          yield self
+          self
+        rescue StandardError => e
+          error!(e)
+          raise
+        ensure
+          complete(**extra)
+        end
+      end
+
+      # Checkpoint the Covered Experience.
+      #
+      # @param extra [Hash] Additional data to include in the log event
+      # @raise [UnstartedError] If the experience has not been started and RAILS_ENV is development or test.
+      # @return [self]
+      def checkpoint(**extra)
+        return unless ensure_started!
+
+        @checkpoint_time = Time.now.utc
+        checkpoint_counter.increment(checkpoint: "intermediate")
+        log_event("intermediate", **extra)
+
+        self
+      end
+
+      # Complete the Covered Experience.
+      #
+      # @param extra [Hash] Additional data to include in the log event
+      # @raise [UnstartedError] If the experience has not been started and RAILS_ENV is development or test.
+      # @return [self]
+      def complete(**extra)
+        return unless ensure_started!
+
+        begin
+          @end_time = Time.now.utc
+        ensure
+          checkpoint_counter.increment(checkpoint: "end")
+          total_counter.increment(error: has_error?)
+          apdex_counter.increment(success: apdex_success?) unless has_error?
+          log_event("end", **extra)
+        end
+
+        self
+      end
+
+      # Marks the experience as failed with an error
+      #
+      # @param error [StandardError, String] The error that caused the experience to fail.
+      # @return [self]
+      def error!(error)
+        @error = error
+        self
+      end
+
+      def has_error?
+        !!@error
+      end
+
+      private
+
+      def base_labels
+        @base_labels ||= @definition.to_h.slice(:covered_experience, :feature_category, :urgency)
+      end
+
+      def ensure_started!
+        return @start_time unless @start_time.nil?
+
+        err = UnstartedError.new("Covered Experience #{@definition.covered_experience} not started")
+
+        warn(err)
+        raise(err) if %w[development test].include?(ENV['RAILS_ENV'])
+      end
+
+      def urgency_threshold
+        URGENCY_THRESHOLDS_IN_SECONDS[@definition.urgency.to_sym]
+      end
+
+      def elapsed_time
+        last_time = @end_time || @checkpoint_time || @start_time
+        last_time - @start_time
+      end
+
+      def apdex_success?
+        elapsed_time <= urgency_threshold
+      end
+
+      def checkpoint_counter
+        @checkpoint_counter ||= Labkit::Metrics::Client.counter(
+          :gitlab_covered_experience_checkpoint_total,
+          'Total checkpoints for covered experiences',
+          base_labels
+        )
+      end
+
+      def total_counter
+        @total_counter ||= Labkit::Metrics::Client.counter(
+          :gitlab_covered_experience_total,
+          'Total covered experience events (success/failure)',
+          base_labels
+        )
+      end
+
+      def apdex_counter
+        @apdex_counter ||= Labkit::Metrics::Client.counter(
+          :gitlab_covered_experience_apdex_total,
+          'Total covered experience apdex events',
+          base_labels
+        )
+      end
+
+      def log_event(event_type, **extra)
+        log_data = build_log_data(event_type, **extra)
+        logger.info(log_data)
+      end
+
+      def build_log_data(event_type, **extra)
+        log_data = {
+          checkpoint: event_type,
+          covered_experience: @definition.covered_experience,
+          feature_category: @definition.feature_category,
+          urgency: @definition.urgency,
+          start_time: @start_time,
+          checkpoint_time: @checkpoint_time,
+          end_time: @end_time,
+          elapsed_time_s: elapsed_time,
+          urgency_threshold_s: urgency_threshold
+        }
+        log_data.merge!(extra) if extra
+
+        if has_error?
+          log_data[:error] = true
+          log_data[:error_message] = @error.inspect
+        end
+
+        log_data.compact!
+
+        log_data
+      end
+
+      def warn(exception)
+        logger.warn(component: self.class.name, message: exception.message)
+      end
+
+      def logger
+        Labkit::CoveredExperience.configuration.logger
+      end
+    end
+  end
+end

data/lib/labkit/covered_experience/null.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Labkit
+  module CoveredExperience
+    # Fakes Labkit::CoveredExperience::Experience.
+    class Null
+      include Singleton
+
+      attr_reader :id, :description, :feature_category, :urgency
+
+      def start(*_args)
+        yield self if block_given?
+        self
+      end
+
+      def push_attributes!(*_args) = self
+      def checkpoint(*_args) = self
+      def complete(*_args) = self
+      def error!(*_args) = self
+    end
+  end
+end

data/lib/labkit/covered_experience/registry.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'json-schema'
+require 'pathname'
+require 'yaml'
+
+module Labkit
+  module CoveredExperience
+    Definition = Data.define(:covered_experience, :description, :feature_category, :urgency)
+
+    class Registry
+      extend Forwardable
+
+      SCHEMA_PATH = File.expand_path('../../../config/covered_experiences/schema.json', __dir__)
+
+      def_delegator :@experiences, :empty?
+
+      # @param dir [String, Pathname] Directory path containing YAML file definitions
+      #   Defaults to 'config/covered_experiences' relative to the calling application's root
+      def initialize(dir: File.join("config", "covered_experiences"))
+        @dir = Pathname.new(Dir.pwd).join(dir)
+        @experiences = load_on_demand
+      end
+
+      # Retrieve a definition experience given a covered_experience_id.
+      #
+      # @param covered_experience_id [String, Symbol] Covered experience identifier
+      # @return [Experience, nil] An experience if present, otherwise nil
+      def [](covered_experience_id)
+        @experiences[covered_experience_id.to_s]
+      end
+
+      private
+
+      # Initialize a hash that loads experiences on-demand
+      #
+      # @return [Hash] Hash with lazy loading behavior
+      def load_on_demand
+        unless readable_dir?
+          warn("Directory not readable: #{@dir}")
+          return {}
+        end
+
+        Hash.new do |result, experience_id|
+          experience = load_experience(experience_id.to_s)
+          # we also store nil to memoize the value and avoid triggering load_experience again
+          result[experience_id.to_s] = experience
+        end
+      end
+
+      # Load a covered experience definition.
+      #
+      # @param experience_id [String] Experience identifier
+      # @return [Experience, nil] Loaded experience or nil if not found/invalid
+      def load_experience(experience_id)
+        file_path = @dir.join("#{experience_id}.yml")
+
+        unless file_path.exist?
+          warn("Invalid Covered Experience definition: #{experience_id}")
+          return nil
+        end
+
+        read_experience(file_path, experience_id)
+      end
+
+      def readable_dir?
+        @dir.exist? && @dir.directory? && @dir.readable?
+      end
+
+      # Read and validate a definition experience file
+      #
+      # @param file_path [Pathname] Path to the definition file
+      # @param experience_id [String] Expected experience ID
+      # @return [Experience, nil] Parsed experience or nil if invalid
+      def read_experience(file_path, experience_id)
+        content = YAML.safe_load(file_path.read)
+        return nil unless content.is_a?(Hash)
+
+        errors = JSON::Validator.fully_validate(schema, content)
+        return Definition.new(covered_experience: experience_id, **content) if errors.empty?
+
+        warn("Invalid schema for #{file_path}")
+
+        nil
+      rescue Psych::SyntaxError => e
+        warn("Invalid definition file #{file_path}: #{e.message}")
+      rescue StandardError => e
+        warn("Unexpected error processing #{file_path}: #{e.message}")
+      end
+
+      def schema
+        @schema ||= JSON.parse(File.read(SCHEMA_PATH))
+      end
+
+      def warn(message)
+        logger.warn(component: self.class.name, message: message)
+      end
+
+      def logger
+        Labkit::CoveredExperience.configuration.logger
+      end
+    end
+  end
+end