gitlab-labkit 0.37.0 → 0.39.0

data/README.md

@@ -1,4 +1,4 @@
-# LabKit-Ruby 🔬🔬🔬🔬🔬
+# LabKit-Ruby 🔬🔬🔬🔬🔬
 
 LabKit-Ruby is minimalist library to provide functionality for Ruby services at GitLab.
 
@@ -22,6 +22,7 @@ LabKit-Ruby provides functionality in a number of areas:
 1. `Labkit::Correlation` For accessing the correlation id. (Generated and propagated by `Labkit::Context`)
 1. `Labkit::FIPS` for checking for FIPS mode and using FIPS-compliant algorithms.
 1. `Labkit::Logging` for sanitizing log messages.
+1. `Labkit::Metrics` for metrics. More on the [README](./lib/labkit/metrics/README.md).
 1. `Labkit::Tracing` for handling and propagating distributed traces.
 
 ## Developing
@@ -40,8 +41,6 @@ $ # Run tests, linters
 $ bundle exec rake verify
 ```
 
-Note that LabKit-Ruby uses the [`rufo`](https://github.com/ruby-formatter/rufo) for auto-formatting. Please run `bundle exec rake fix` to auto-format your code before pushing.
-
 Please also review the [development section of the LabKit (go) README](https://gitlab.com/gitlab-org/labkit#developing-labkit) for details of the LabKit architectural philosophy.
 
 To work on some of the scripts we use for releasing a new version,
@@ -51,7 +50,7 @@ make sure to add a new `.env.sh`.
 cp .env.example.sh .env.sh`
 ```
 
-Inside `.env.sh`, add a personal acccess token for the `GITLAB_TOKEN`
+Inside `.env.sh`, add a personal acccess token for the `CHANGELOG_GITLAB_TOKEN`
 environment variable. Next source the file:
 
 ```console

data/Rakefile

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "bundler/gem_tasks"
-require "rufo"
 
 require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
@@ -11,27 +10,9 @@ RuboCop::RakeTask.new(:rubocop) do |task|
   task.options = %w[--parallel]
 end
 
-desc "Alias for `rake rufo:run`"
-task :format => ["rufo:run"]
-
-namespace :rufo do
-  require "rufo"
-
-  def rufo_command(*switches, rake_args)
-    files_or_dirs = rake_args[:files_or_dirs] || "."
-    args = switches + files_or_dirs.split(" ")
-    Rufo::Command.run(args)
-  end
-
-  desc "Format Ruby code in current directory"
-  task :run, [:files_or_dirs] do |_task, rake_args|
-    rufo_command(rake_args)
-  end
-
-  desc "Check that no formatting changes are produced"
-  task :check, [:files_or_dirs] do |_task, rake_args|
-    rufo_command("--check", rake_args)
-  end
+desc "update the rubocop todo config"
+RuboCop::RakeTask.new("rubocop:config") do |task|
+  task.options = %w[--auto-gen-config]
 end
 
 desc "Generate test protobuf stubs"
@@ -40,8 +21,17 @@ task :gen_test_proto do
   Rufo::Command.run(["spec/support/grpc_service/test_pb.rb", "spec/support/grpc_service/test_services_pb.rb"])
 end
 
-task :fix => %w[rufo:run rubocop:auto_correct]
+task :fix => ["rubocop:autocorrect"]
 
-task :verify => %w[spec rufo:check rubocop]
+task :verify => %w[spec rubocop]
 
 task :default => %w[verify build]
+
+desc "Start an IRB console with gem pre-loaded"
+task :console do
+  $LOAD_PATH.unshift(File.expand_path("lib", __dir__))
+  require "irb"
+  require "gitlab-labkit"
+  ARGV.clear
+  IRB.start
+end

data/gitlab-labkit.gemspec

@@ -5,7 +5,8 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name = "gitlab-labkit"
-  spec.version = `git describe --tags`.chomp.gsub(/^v/, "")
+  version = ENV["CI_COMMIT_TAG"] || `git describe --tag --match "v[0-9]*.[0-9]*.[0-9]*"`
+  spec.version = version.chomp.gsub(/^v/, "")
   spec.authors = ["Andrew Newdigate"]
   spec.email = ["andrew@gitlab.com"]
 
@@ -16,32 +17,35 @@ Gem::Specification.new do |spec|
 
   spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|tools)/}) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = ">= 2.6.0"
+  spec.required_ruby_version = "~> 3.2"
 
   # Please maintain alphabetical order for dependencies
   spec.add_runtime_dependency "actionpack", ">= 5.0.0", "< 8.1.0"
   spec.add_runtime_dependency "activesupport", ">= 5.0.0", "< 8.1.0"
   spec.add_runtime_dependency "grpc", ">= 1.62" # Be sure to update the "grpc-tools" dev_dependency too
+  spec.add_runtime_dependency "google-protobuf", "~> 3" # Keep the major version to 3 until we update the `grpc` gem
   spec.add_runtime_dependency "jaeger-client", "~> 1.1.0"
   spec.add_runtime_dependency "opentracing", "~> 0.4"
-  spec.add_runtime_dependency "pg_query", ">= 5.1.0", "< 7.0"
+  spec.add_runtime_dependency "pg_query", ">= 6.1.0", "< 7.0"
+  spec.add_runtime_dependency "prometheus-client-mmap", "~> 1.2.9"
   spec.add_runtime_dependency "redis", "> 3.0.0", "< 6.0.0"
 
   # Please maintain alphabetical order for dev dependencies
   spec.add_development_dependency "excon", "~> 0.78.1"
   spec.add_development_dependency "faraday", "~> 1.10.3"
   spec.add_development_dependency "gitlab-dangerfiles", "~> 2.11.0"
-  spec.add_development_dependency "gitlab-styles", "~> 6.2.0"
+  spec.add_development_dependency "gitlab-styles", "~> 13.0.2"
   spec.add_development_dependency "grpc-tools", ">= 1.62"
-  spec.add_development_dependency "httparty", "~> 0.17.3"
-  spec.add_development_dependency "httpclient", "~> 2.8.3"
+  spec.add_development_dependency "httparty", "~> 0.22.0"
+  spec.add_development_dependency "httpclient", "~> 2.9.0"
+  spec.add_development_dependency "irb", "~> 1.15.2"
   spec.add_development_dependency "pry", "~> 0.12"
+  spec.add_development_dependency "pry-byebug", "~> 3.11"
   spec.add_development_dependency "rack", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 12.3"
+  spec.add_development_dependency "rake", "~> 13.2"
   spec.add_development_dependency "rest-client", "~> 2.1.0"
   spec.add_development_dependency "rspec", "~> 3.12.0"
   spec.add_development_dependency "rspec-parameterized", "~> 1.0"
-  spec.add_development_dependency "rufo", "0.9.0"
   spec.add_development_dependency "sidekiq", ">= 5.2", "< 7"
   spec.add_development_dependency "webrick", "~> 1.7.0"
 end

data/lib/gitlab-labkit.rb

@@ -12,6 +12,7 @@ module Labkit
   autoload :FIPS, "labkit/fips"
   autoload :Tracing, "labkit/tracing"
   autoload :Logging, "labkit/logging"
+  autoload :Metrics, "labkit/metrics"
   autoload :Middleware, "labkit/middleware"
 
   # Publishers to publish notifications whenever a HTTP reqeust is made.

data/lib/labkit/metrics/README.md

@@ -0,0 +1,98 @@
+# Labkit::Metrics
+
+## Usage
+
+```ruby
+# create a new counter metric and returns it
+http_requests = Labkit::Metrics::Client.counter(:http_requests, 'A counter of HTTP requests made')
+# start using the counter
+http_requests.increment
+
+# resets the registry and reinitializes all metrics files
+Labkit::Metrics::Client.reset!
+
+# retrieves the metric (be it a counter, gauge, histogram, summary)
+http_requests = Labkit::Metrics::Client.get(:http_requests)
+```
+
+### Counter
+
+```ruby
+counter = Labkit::Metrics::Client.counter(:service_requests_total, '...')
+
+# increment the counter for a given label set
+counter.increment({ service: 'foo' })
+
+# increment by a given value
+counter.increment({ service: 'bar' }, 5)
+
+# get current value for a given label set
+counter.get({ service: 'bar' })
+# => 5
+```
+
+### Gauge
+
+```ruby
+gauge = Labkit::Metrics::Client.gauge(:room_temperature_celsius, '...')
+
+# set a value
+gauge.set({ room: 'kitchen' }, 21.534)
+
+# retrieve the current value for a given label set
+gauge.get({ room: 'kitchen' })
+# => 21.534
+```
+
+### Histogram
+
+```ruby
+histogram = Labkit::Metrics::Client.histogram(:service_latency_seconds, '...')
+
+# record a value
+histogram.observe({ service: 'users' }, Benchmark.realtime { service.call(arg) })
+
+# retrieve the current bucket values
+histogram.get({ service: 'users' })
+# => { 0.005 => 3, 0.01 => 15, 0.025 => 18, ..., 2.5 => 42, 5 => 42, 10 => 42 }
+```
+
+### Summary
+
+```ruby
+summary = Labkit::Metrics::Client.summary(:service_latency_seconds, '...')
+
+# record a value
+summary.observe({ service: 'database' }, Benchmark.realtime { service.call() })
+
+# retrieve the current quantile values
+summary.get({ service: 'database' })
+# => { 0.5 => 0.1233122, 0.9 => 3.4323, 0.99 => 5.3428231 }
+```
+
+## Rack middleware
+
+```ruby
+# config.ru
+
+require 'rack'
+require 'labkit/metrics/rack_exporter'
+
+use Labkit::Metrics::RackExporter
+
+run ->(env) { [200, {'Content-Type' => 'text/html'}, ['OK']] }
+```
+
+## Configuration
+
+```ruby
+# config/initializers/metrics.rb
+
+Labkit::Metrics::Client.reinitialize_on_pid_change(force: true)
+
+Labkit::Metrics::Client.configure do |config|
+  config.logger = Gitlab::AppLogger
+  config.multiprocess_files_dir = 'tmp/prometheus_multiproc_dir'
+  config.pid_provider = ::Prometheus::PidProvider.method(:worker_id)
+end
+```

data/lib/labkit/metrics/client.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "singleton"
+require 'concurrent-ruby'
+require "prometheus/client"
+require "labkit/metrics/registry"
+require "labkit/metrics/null"
+
+module Labkit
+  module Metrics
+    InvalidLabelSet = Class.new(RuntimeError)
+
+    # A thin wrapper around Prometheus::Client from the prometheus-client-mmap gem
+    # https://gitlab.com/gitlab-org/ruby/gems/prometheus-client-mmap
+    class Client
+      include Singleton
+      extend Forwardable
+
+      def_delegators :wrapped_client, :configure, :reinitialize_on_pid_change, :configuration
+
+      def initialize
+        @enabled = Concurrent::AtomicBoolean.new(true)
+      end
+
+      def wrapped_client
+        @client ||= ::Prometheus::Client
+      end
+
+      def disable!
+        @enabled.make_false
+      end
+
+      def enable!
+        @enabled.make_true
+      end
+
+      def enabled?
+        @enabled.true? && metrics_folder_present?
+      end
+
+      def safe_provide_metric(metric_type, name, *args)
+        return Null.instance unless enabled?
+
+        Registry.safe_register(metric_type, name, *args)
+      end
+
+      def metrics_folder_present?
+        dir = configuration.multiprocess_files_dir
+        dir && Dir.exist?(dir) && File.writable?(dir)
+      end
+
+      def counter(name, docstring, base_labels = {})
+        safe_provide_metric(:counter, name, docstring, base_labels)
+      end
+
+      def summary(name, docstring, base_labels = {})
+        safe_provide_metric(:summary, name, docstring, base_labels)
+      end
+
+      def histogram(
+        name, docstring, base_labels = {},
+        buckets = Prometheus::Client::Histogram::DEFAULT_BUCKETS)
+        safe_provide_metric(:histogram, name, docstring, base_labels, buckets)
+      end
+
+      def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all)
+        safe_provide_metric(:gauge, name, docstring, base_labels, multiprocess_mode)
+      end
+
+      def reset!
+        Registry.reset!
+      end
+
+      def get(metric_name)
+        Registry.get(metric_name)
+      end
+
+      class << self
+        extend Forwardable
+
+        def_delegators :instance,
+          :enable!, :disable!, :counter, :gauge, :histogram, :summary, :reset!, :enabled?,
+          :configure, :reinitialize_on_pid_change, :configuration, :get
+
+        private :instance
+      end
+    end
+  end
+end

data/lib/labkit/metrics/null.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Metrics
+    # Fakes Prometheus::Client::Metric and all derived metrics.
+    # Explicitly avoiding meta-programming to make interface more obvious to interact with.
+    class Null
+      include Singleton
+
+      attr_reader :name, :docstring, :base_labels
+
+      def get(*args); end
+      def set(*args); end
+      def increment(*args); end
+      def decrement(*args); end
+      def observe(*args); end
+      def values(*args); end
+    end
+  end
+end

data/lib/labkit/metrics/rack_exporter.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "prometheus/client/rack/exporter"
+
+module Labkit
+  module Metrics
+    # A wrapper around the Rack exporter middleware provided by
+    # prometheus-client-mmap gem
+    # https://gitlab.com/gitlab-org/ruby/gems/prometheus-client-mmap
+    class RackExporter < Prometheus::Client::Rack::Exporter; end
+  end
+end

data/lib/labkit/metrics/registry.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "prometheus/client"
+
+module Labkit
+  module Metrics
+    InvalidMetricType = Class.new(StandardError)
+
+    # A thin wrapper around Prometheus::Client::Registry.
+    # It provides a thread-safe way to register metrics with the Prometheus registry.
+    class Registry
+      class << self
+        INIT_REGISTRY_MUTEX = Mutex.new
+        REGISTER_MUTEX = Mutex.new
+
+        # Registers a metric with the Prometheus registry in a thread-safe manner.
+        # If the metric already exists, it returns the existing metric.
+        # If the metric does not exist, it creates a new one.
+        # Each metric-name is only registered once for a type (counter, gauge, histogram, summary),
+        # even if multiple threads attempt to register the same metric simultaneously.
+        #
+        # @param metric_type [Symbol] The type of metric to register (:counter, :gauge, :histogram, :summary)
+        # @param name [Symbol, String] The name of the metric
+        # @param args [Array] Additional arguments to pass to the metric constructor
+        # @return [Prometheus::Client::Metric] The registered metric
+        # @raise [InvalidMetricType] If the metric_type is not supported
+        #
+        # @example
+        #   # Register a counter
+        #   counter = Registry.safe_register(:counter, :http_requests_total, 'Total HTTP requests')
+        def safe_register(metric_type, name, *args)
+          REGISTER_MUTEX.synchronize do
+            get(name) || wrapped_registry.method(metric_type).call(name, *args)
+          end
+        end
+
+        # Cleans up the Prometheus registry and resets it to a new state.
+        def reset!
+          INIT_REGISTRY_MUTEX.synchronize do
+            Prometheus::Client.cleanup!
+            Prometheus::Client.reset!
+            @registry = nil
+          end
+        end
+
+        # Returns the metric for the given name from the Prometheus registry.
+        #
+        # @param metric_name [Symbol, String] The name of the metric
+        # @return [Prometheus::Client::Metric, nil] The registered metric or nil if it does not exist
+        def get(metric_name)
+          wrapped_registry.get(metric_name)
+        end
+
+        private
+
+        def wrapped_registry
+          @registry ||= init_registry
+        end
+
+        def init_registry
+          # Prometheus::Client.registry initializes a new registry with an underlying hash
+          # storing metrics and a mutex synchronizing the writes to that hash.
+          # This means we need to make sure we only build one registry, for accessing from within Labkit.
+          INIT_REGISTRY_MUTEX.synchronize { Prometheus::Client.registry }
+        end
+      end
+    end
+  end
+end

data/lib/labkit/metrics.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "prometheus/client/formats/text"
+
+module Labkit
+  # Metrics provides functionality for producing metrics
+  module Metrics
+    autoload :Client, "labkit/metrics/client"
+    autoload :RackExporter, "labkit/metrics/rack_exporter"
+    autoload :Null, "labkit/metrics/null"
+
+    class << self
+      def prometheus_metrics_text
+        dir = Client.configuration.multiprocess_files_dir
+        ::Prometheus::Client::Formats::Text.marshal_multiprocess(dir)
+      end
+    end
+  end
+end

data/renovate.json

@@ -0,0 +1,7 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "gitlab>gitlab-com/gl-infra/common-ci-tasks:renovate-common",
+    "gitlab>gitlab-com/gl-infra/common-ci-tasks:renovate-truncated-versions"
+  ]
+}

data/scripts/install-asdf-plugins.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Vendored from https://gitlab.com/gitlab-com/gl-infra/common-template-copier
+# Consider contributing upstream when updating this file
+
+# This file is deprecated: going forward running `mise install` should be sufficient.
+
+set -euo pipefail
+IFS=$'\n\t'
+
+echo >&2 -e "2024-08-07: this file is deprecated: going forward, simply run 'mise install' to install plugins."
+echo >&2 -e "Recommended reading: https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/docs/developer-setup.md"
+
+mise install

data/scripts/prepare-dev-env.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Vendored from https://gitlab.com/gitlab-com/gl-infra/common-template-copier
+# Consider contributing upstream when updating this file
+
+set -euo pipefail
+
+# ---------------------------------------------------------
+# This script will prepare your development environment
+# while working on this project. Run it after cloning this
+# project.
+#
+# It's recommended that you review
+# https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/docs/developer-setup.md
+# first.
+# ---------------------------------------------------------
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+warn() {
+  echo >&2 -e "${1-}"
+  echo >&2 -e "Recommended reading: https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/docs/developer-setup.md"
+}
+
+if command -v mise >/dev/null; then
+  echo >&2 -e "mise installed..."
+elif command -v rtx >/dev/null; then
+  warn "⚠️ 2024-01-02: 'rtx' has changed to 'mise'; please replace 'rtx' with 'mise'"
+  exit 1
+elif [[ -n ${ASDF_DIR-} ]]; then
+  warn "⚠️ 2024-08-07: 'asdf' is no longer supported; please uninstall and replace with 'mise'"
+  exit 1
+else
+  warn "mise is not installed."
+  exit 1
+fi
+
+# Do some validation to ensure that the environment is not misconfigured, as this may
+# save a bunch of debugging effort down the line.
+
+# Detect Rosetta 2
+if [[ $(uname -m) == "arm64" ]] && [[ $(uname -p) == "x86_64" ]]; then
+  echo "This shell is running in Rosetta emulating x86_64. Please use native mode Apple Silicon." >&2
+  echo "For help visit https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/docs/developer-setup.md" >&2
+  exit 1
+fi
+
+# Detect ancient versions of bash
+if ((BASH_VERSINFO[0] < 4)); then
+  echo "You're running bash < v4.0.0. Please upgrade to a newer version." >&2
+  echo "For help visit https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/docs/developer-setup.md" >&2
+  exit 1
+fi
+
+# install mise/asdf dependencies
+echo "installing required plugins with mise install.."
+mise plugins update
+mise install
+eval "$(mise activate bash)"
+
+# pre-commit is optional
+if command -v pre-commit &>/dev/null; then
+  echo "running pre-commit install..."
+  pre-commit install
+  pre-commit install-hooks
+  pre-commit install --hook-type commit-msg
+else
+  warn "pre-commit is not installed. Skipping."
+fi

data/scripts/update-asdf-version-variables.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Vendored from https://gitlab.com/gitlab-com/gl-infra/common-template-copier
+# Consider contributing upstream when updating this file
+
+# See the README.md for details of how this script works
+
+set -euo pipefail
+IFS=$'\n\t'
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+export LANG=C LC_ALL=C
+
+generate() {
+  sort "${ROOT_DIR}/.tool-versions" |
+    awk '
+    BEGIN {
+      print "# DO NOT MANUALLY EDIT; Run ./scripts/update-asdf-version-variables.sh to update this";
+      print "variables:"
+    }
+    {
+      if (!/^#/ && $1 != "" && $2 != "system") {
+        gsub("-", "_", $1);
+        print "    GL_ASDF_" toupper($1) "_VERSION: \"" $2 "\""
+      }
+    }
+    '
+}
+
+generate >"${ROOT_DIR}/.gitlab-ci-asdf-versions.yml"