gitlab-labkit 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4260cb369ba6b4c22a6e11a74d659eb9b4167a1686ca0cdb44790257bd998424
-  data.tar.gz: 9dafed39c31c650cddd8db4d55e130f788ee6f5a7ba803195d194c57d4e90632
+  metadata.gz: c306478521702991b9c5ba1d80e6ab1c3bcb42b70562890a48204ab9e320c6b8
+  data.tar.gz: 5b4fdf319fa79405c89c0de7d91617602a6aec6bf784223a6d8e86830c6900ff
 SHA512:
-  metadata.gz: 391b5320921ab7222045400828e21c8b92f28764bae9ad2bd4cd1525ce37099eeed5f9ff4af942c5a74233d9ce5819907ca77d475b3f575d281baedb08640ee1
-  data.tar.gz: '00386cc726461cae68fa4648fd43a27eb6afeacce77e61348ce6244b0c1b4c4fd9bceb252e7c5d5464cbb1d6062f6ae6a8fb61bb013ed109d4361447ba172dcd'
+  metadata.gz: f88db3bf552afa6f20e95e8760d91b97149d41e4b1cfe995c0e64bdc852ca0490713ba52a5bdcadebea1c827b83b344df61695386741c37d2dd8458243bd0144
+  data.tar.gz: a1ab50bd164cd46da510b03ac53aa3e148a6b8720611ef380b288d282072446b06772c43777c019da4fd93d5d9b1e1f197704fc36bf0b6e47985fc35535e2a9b

data/.ruby-version

@@ -1 +1 @@
-ruby-2.5.3
+2.6.3

data/gitlab-labkit.gemspec

@@ -28,11 +28,13 @@ Gem::Specification.new do |spec|
 
   # Please maintain alphabetical order for dev dependencies
   spec.add_development_dependency "grpc-tools", "~> 1.19"
+  spec.add_development_dependency "pry", "~> 0.12"
   spec.add_development_dependency "rack", "~> 2.0"
   spec.add_development_dependency "rake", "~> 12.3"
-  spec.add_development_dependency "rspec", "~> 3.6.0"
+  spec.add_development_dependency "rspec", "~> 3.8.0"
   spec.add_development_dependency "rspec-parameterized", "~> 0.4"
   spec.add_development_dependency "rubocop", "~> 0.65.0"
   spec.add_development_dependency "rubocop-rspec", "~> 1.22.1"
   spec.add_development_dependency "rufo", "~> 0.6"
+  spec.add_development_dependency "sidekiq", "~> 5.2.7"
 end

data/lib/gitlab-labkit.rb

@@ -8,8 +8,10 @@ require "active_support/all"
 # observability.
 module Labkit
   autoload :Correlation, "labkit/correlation"
+  autoload :Context, "labkit/context"
   autoload :Tracing, "labkit/tracing"
   autoload :Logging, "labkit/logging"
+  autoload :Middleware, "labkit/middleware"
 end
 
 # rubocop:enable Naming/FileName

data/lib/labkit/context.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module Labkit
+  # A context can be used to provide structured information on what resources
+  # GitLab is working on within a service.
+  # The currently supported keys are defined in the `KNOWN_KEYS` constant.
+  #
+  # Values can be provided by passing a hash. If one of the values is a Proc
+  # the proc will only be called when the value is actually needed.
+  #
+  # Multiple contexts can be nested, the nested context will inherit the values
+  # from the closest outer one.
+  # All contexts will have the same correlation id.
+  #
+  # Usage:
+  #   Labkit::Context.with_context(user: 'username', root_namespace: -> { get_root_namespace } do |context|
+  #     logger.info(context.to_h)
+  #   end
+  #
+  class Context
+    LOG_KEY = "meta"
+    CORRELATION_ID_KEY = "correlation_id"
+    RAW_KEYS = [CORRELATION_ID_KEY].freeze
+    KNOWN_KEYS = %w[user project root_namespace].freeze
+
+    class << self
+      def with_context(attributes = {})
+        context = push(attributes)
+
+        begin
+          yield(context)
+        ensure
+          pop(context)
+        end
+      end
+
+      def push(new_attributes = {})
+        new_context = current&.merge(new_attributes) || new(new_attributes)
+
+        contexts.push(new_context)
+
+        new_context
+      end
+
+      def pop(context)
+        contexts.pop while contexts.include?(context)
+      end
+
+      def correlation_id
+        contexts.last&.correlation_id
+      end
+
+      def current
+        contexts.last
+      end
+
+      def log_key(key)
+        key = key.to_s
+        return key if RAW_KEYS.include?(key)
+        return key if key.starts_with?("#{LOG_KEY}.")
+
+        "#{LOG_KEY}.#{key}"
+      end
+
+      def known_log_keys
+        @known_log_keys ||= (KNOWN_KEYS.map(&method(:log_key)) + RAW_KEYS).freeze
+      end
+
+      private
+
+      def contexts
+        Thread.current[:labkit_contexts] ||= []
+      end
+    end
+
+    def initialize(values = {})
+      @data = {}
+
+      assign_attributes(values)
+    end
+
+    def merge(new_attributes)
+      new_context = self.class.new(data.dup)
+      new_context.assign_attributes(new_attributes)
+
+      new_context
+    end
+
+    def to_h
+      expand_data
+    end
+
+    def correlation_id
+      data[CORRELATION_ID_KEY]
+    end
+
+    protected
+
+    def assign_attributes(attributes)
+      attributes = attributes.transform_keys(&method(:log_key))
+      attributes = attributes.slice(*known_log_keys)
+
+      data.merge!(attributes)
+
+      # Remove keys that had their values set to `nil` in the new attributes
+      data.keep_if { |_, value| value.present? }
+
+      # Assign a correlation if it was missing in the first context or when
+      # explicitly removed
+      data[CORRELATION_ID_KEY] ||= new_id
+
+      data
+    end
+
+    private
+
+    delegate :log_key, :known_log_keys, to: :class
+
+    attr_reader :data
+
+    def expand_data
+      data.transform_values do |value|
+        value = value.respond_to?(:call) ? value.call : value
+
+        value.presence
+      end.compact
+    end
+
+    def new_id
+      SecureRandom.hex
+    end
+  end
+end

data/lib/labkit/correlation/correlation_id.rb

@@ -5,38 +5,21 @@ module Labkit
     # CorrelationId module provides access the Correlation-ID
     # of the current request
     module CorrelationId
-      LOG_KEY = "correlation_id"
+      LOG_KEY = Labkit::Context::CORRELATION_ID_KEY
 
       class << self
-        def use_id(correlation_id, &_blk)
-          # always generate a id if null is passed
-          correlation_id ||= new_id
-
-          ids.push(correlation_id || new_id)
-
-          begin
-            yield(current_id)
-          ensure
-            ids.pop
+        def use_id(correlation_id)
+          Labkit::Context.with_context(LOG_KEY => correlation_id) do |context|
+            yield(context.correlation_id)
           end
         end
 
         def current_id
-          ids.last
+          Labkit::Context.correlation_id
         end
 
         def current_or_new_id
-          current_id || new_id
-        end
-
-        private
-
-        def ids
-          Thread.current[:correlation_id] ||= []
-        end
-
-        def new_id
-          SecureRandom.uuid
+          current_id || Labkit::Context.push.correlation_id
         end
       end
     end

data/lib/labkit/correlation/grpc.rb

@@ -4,6 +4,7 @@ module Labkit
   module Correlation
     # The GRPC module contains functionality for instrumenting GRPC calls
     module GRPC
+      autoload :ClientInterceptor, "labkit/correlation/grpc/client_interceptor"
       autoload :GRPCCommon, "labkit/correlation/grpc/grpc_common"
       autoload :ServerInterceptor, "labkit/correlation/grpc/server_interceptor"
     end

data/lib/labkit/correlation/grpc/client_interceptor.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# Disable the UnusedMethodArgument linter, since we need to declare the kwargs
+# in the methods, but we don't actually use them.
+# rubocop:disable Lint/UnusedMethodArgument
+
+require "grpc"
+
+module Labkit
+  module Correlation
+    module GRPC
+      # ClientInterceptor is used to inject the correlation_id into the metadata
+      # or a GRPC call for onward propagation to the server
+      class ClientInterceptor < ::GRPC::ClientInterceptor
+        include Labkit::Correlation::GRPC::GRPCCommon
+        include Singleton
+
+        def request_response(request:, call:, method:, metadata:)
+          inject_correlation_id_into_metadata(metadata)
+
+          yield
+        end
+
+        def client_streamer(requests:, call:, method:, metadata:)
+          inject_correlation_id_into_metadata(metadata)
+
+          yield
+        end
+
+        def server_streamer(request:, call:, method:, metadata:)
+          inject_correlation_id_into_metadata(metadata)
+
+          yield
+        end
+
+        def bidi_streamer(requests:, call:, method:, metadata:)
+          inject_correlation_id_into_metadata(metadata)
+
+          yield
+        end
+
+        private
+
+        def inject_correlation_id_into_metadata(metadata, &block)
+          metadata[CORRELATION_METADATA_KEY] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id
+        end
+      end
+    end
+  end
+end
+
+# rubocop:enable Lint/UnusedMethodArgument

data/lib/labkit/middleware.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Labkit
+  # Adds middlewares for using in rack and sidekiq
+  module Middleware
+    autoload :Rack, "labkit/middleware/rack"
+    autoload :Sidekiq, "labkit/middleware/sidekiq"
+  end
+end

data/lib/labkit/middleware/rack.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "action_dispatch"
+
+module Labkit
+  module Middleware
+    # This is a rack middleware to be inserted in GitLab-rails
+    # It makes sure that there's always a root context containing the correlation
+    # id.
+    # Since this context always get's cleaned up by this middleware, we can be
+    # sure that any nested contexts will also be cleaned up.
+    class Rack
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        Labkit::Context.with_context(Labkit::Context::CORRELATION_ID_KEY => correlation_id(env)) do
+          @app.call(env)
+        end
+      end
+
+      private
+
+      def correlation_id(env)
+        request(env).request_id
+      end
+
+      def request(env)
+        ActionDispatch::Request.new(env)
+      end
+    end
+  end
+end

data/lib/labkit/middleware/sidekiq.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Middleware
+    # Middleware for sidekiq
+    module Sidekiq
+      autoload :Client, "labkit/middleware/sidekiq/client"
+      autoload :Server, "labkit/middleware/sidekiq/server"
+    end
+  end
+end

data/lib/labkit/middleware/sidekiq/client.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Middleware
+    module Sidekiq
+      # This middleware for Sidekiq-client wraps scheduling jobs in a context
+      # The context will also be added to the sidekiq job in redis so it can
+      # be reinstantiated by Sidekiq-server when running the job.
+      class Client
+        def call(_worker_class, job, _queue, _redis_pool)
+          Labkit::Context.with_context do |context|
+            job.merge!(context.to_h)
+
+            yield
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/middleware/sidekiq/server.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Middleware
+    module Sidekiq
+      # This middleware for Sidekiq-client uses the values stored on a job to
+      # reinstantiate a context in which the job will run.
+      class Server
+        def call(_worker_class, job, _queue)
+          Labkit::Context.with_context(job) do |_context|
+            yield
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-labkit
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Andrew Newdigate
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-04 00:00:00.000000000 Z
+date: 2019-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -126,6 +126,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.19'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -160,14 +174,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.6.0
+        version: 3.8.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.6.0
+        version: 3.8.0
 - !ruby/object:Gem::Dependency
   name: rspec-parameterized
   requirement: !ruby/object:Gem::Requirement
@@ -224,6 +238,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.2.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.2.7
 description: 
 email:
 - andrew@gitlab.com
@@ -243,13 +271,20 @@ files:
 - Rakefile
 - gitlab-labkit.gemspec
 - lib/gitlab-labkit.rb
+- lib/labkit/context.rb
 - lib/labkit/correlation.rb
 - lib/labkit/correlation/correlation_id.rb
 - lib/labkit/correlation/grpc.rb
+- lib/labkit/correlation/grpc/client_interceptor.rb
 - lib/labkit/correlation/grpc/grpc_common.rb
 - lib/labkit/correlation/grpc/server_interceptor.rb
 - lib/labkit/logging.rb
 - lib/labkit/logging/sanitizer.rb
+- lib/labkit/middleware.rb
+- lib/labkit/middleware/rack.rb
+- lib/labkit/middleware/sidekiq.rb
+- lib/labkit/middleware/sidekiq/client.rb
+- lib/labkit/middleware/sidekiq/server.rb
 - lib/labkit/tracing.rb
 - lib/labkit/tracing/factory.rb
 - lib/labkit/tracing/grpc.rb