gitlab-labkit 0.9.1 → 0.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c6a4f92530f7662034e9a67cda9b6f1587cce729502a088ef9b6777659b0bdc
-  data.tar.gz: 1f17a19f8e2f7a0f052c4474caaaf0fd1d3a291a77de67b4184000fb12a599de
+  metadata.gz: 3267849e09cf37b1f72c5c7cf0f25ac43066edd06b09ff7b8a3b93f2ee5a4f45
+  data.tar.gz: ae8f6710ada3fc3791ae54cba82a4f8b73c5d67003782285264246032c22328e
 SHA512:
-  metadata.gz: 2a50b5944898571c6fa9038eed5d35ec3d560c684c7363e1779aa8b4ec1363e8533e51667ae868b961562a3d3195cc9ac1f8b27e6e76204290439a735b61f5bc
-  data.tar.gz: d96aeafefa46bb244fc7371165fed6f91e47326d60463f8d25685d50081b63a8651c2b5dee7656094d5fb87085a58dd44459aafc64621b32a083dde3eaf7180e
+  metadata.gz: 33ec95dfe61241476125e718e9ae3d3145ba2d76244076cfbe164dd9b61b841d1e832dc27ef71846476ca32abe1057165f35c2b44abb3ff83bd09693a354f00b
+  data.tar.gz: cb7abc95873cae4e981348c28533f3c894e50b696ed87822cca9b65a6db689a4e0453b83297b667eaa88c7a879e80dfc520e56e75959431a1388bc1db622aaf3

data/README.md

@@ -18,7 +18,8 @@ The changelog is available via [**tagged release notes**](https://gitlab.com/git
 
 LabKit-Ruby provides functionality in three areas:
 
-1. `Labkit::Correlation` for handling and propagating Correlation-IDs.
+1. `Labkit::Context` used for providing context information to log messages.
+1. `Labkit::Correlation` For accessing the correlation id. (Generated and propagated by `Labkit::Context`)
 1. `Labkit::Logging` for sanitizing log messages.
 1. `Labkit::Tracing` for handling and propagating distributed traces.
 
@@ -41,3 +42,16 @@ $ bundle exec rake verify
 Note that LabKit-Ruby uses the [`rufo`](https://github.com/ruby-formatter/rufo) for auto-formatting. Please run `bundle exec rake fix` to auto-format your code before pushing.
 
 Please also review the [development section of the LabKit (go) README](https://gitlab.com/gitlab-org/labkit#developing-labkit) for details of the LabKit architectural philosophy.
+
+### Releasing a new version
+
+Releasing a new version can be done by pushing a new tag, or creating
+it from the
+[interface](https://gitlab.com/gitlab-org/labkit-ruby/-/tags).
+
+A new changelog will automatically be added to the release on Gitlab.
+
+The new version will automatically be published to
+[rubygems](https://rubygems.org/gems/gitlab-labkit) when the pipeline
+for the tag completes. It might take a few minutes before the update
+is available.

data/lib/labkit/context.rb

@@ -21,7 +21,8 @@ module Labkit
     LOG_KEY = "meta"
     CORRELATION_ID_KEY = "correlation_id"
     RAW_KEYS = [CORRELATION_ID_KEY].freeze
-    KNOWN_KEYS = %w[user project root_namespace subscription_plan caller_id].freeze
+    KNOWN_KEYS = %w[user project root_namespace subscription_plan caller_id
+                    related_class].freeze
 
     class << self
       def with_context(attributes = {})

data/lib/labkit/middleware/sidekiq.rb

@@ -6,6 +6,8 @@ module Labkit
     module Sidekiq
       autoload :Client, "labkit/middleware/sidekiq/client"
       autoload :Server, "labkit/middleware/sidekiq/server"
+      autoload :Context, "labkit/middleware/sidekiq/context"
+      autoload :Tracing, "labkit/middleware/sidekiq/tracing"
     end
   end
 end

data/lib/labkit/middleware/sidekiq/client.rb

@@ -3,14 +3,21 @@
 module Labkit
   module Middleware
     module Sidekiq
-      # This middleware for Sidekiq-client wraps scheduling jobs in a context
-      # The context will also be added to the sidekiq job in redis so it can
-      # be reinstantiated by Sidekiq-server when running the job.
+      # This is a wrapper around all the sidekiq client-middleware in labkit
+      # The only middleware that needs to be added to the chain in GitLab-rails
+      #
+      # It uses a new `Sidekiq::Middleware::Chain` to string multiple middlewares
+      # together.
       class Client
-        def call(_worker_class, job, _queue, _redis_pool)
-          Labkit::Context.with_context do |context|
-            job.merge!(context.to_h)
+        def self.chain
+          @chain ||= ::Sidekiq::Middleware::Chain.new do |chain|
+            chain.add Labkit::Middleware::Sidekiq::Context::Client
+            chain.add Labkit::Middleware::Sidekiq::Tracing::Client if Labkit::Tracing.enabled?
+          end
+        end
 
+        def call(*args)
+          self.class.chain.invoke(*args) do
             yield
           end
         end

data/lib/labkit/middleware/sidekiq/context.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Middleware
+    module Sidekiq
+      # This module contains all the sidekiq middleware regarding application
+      # context
+      module Context
+        autoload :Client, "labkit/middleware/sidekiq/context/client"
+        autoload :Server, "labkit/middleware/sidekiq/context/server"
+      end
+    end
+  end
+end

data/lib/labkit/middleware/sidekiq/context/client.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Middleware
+    module Sidekiq
+      module Context
+        # This middleware for Sidekiq-client wraps scheduling jobs in a context
+        # The context will also be added to the sidekiq job in redis so it can
+        # be reinstantiated by Sidekiq-server when running the job.
+        class Client
+          def call(_worker_class, job, _queue, _redis_pool)
+            Labkit::Context.with_context do |context|
+              job.merge!(context.to_h)
+
+              yield
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/middleware/sidekiq/context/server.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Middleware
+    module Sidekiq
+      module Context
+        # This middleware for Sidekiq-client uses the values stored on a job to
+        # reinstantiate a context in which the job will run.
+        class Server
+          def call(_worker_class, job, _queue)
+            data = job.merge(Labkit::Context.log_key(:caller_id) => job["class"])
+
+            Labkit::Context.with_context(data) do |_context|
+              yield
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/middleware/sidekiq/server.rb

@@ -3,13 +3,21 @@
 module Labkit
   module Middleware
     module Sidekiq
-      # This middleware for Sidekiq-client uses the values stored on a job to
-      # reinstantiate a context in which the job will run.
+      # This is a wrapper around all the sidekiq server-middleware in labkit
+      # The only middleware that needs to be added to the chain in GitLab-rails
+      #
+      # It uses a new `Sidekiq::Middleware::Chain` to string multiple middlewares
+      # together.
       class Server
-        def call(_worker_class, job, _queue)
-          data = job.merge(Labkit::Context.log_key(:caller_id) => job["class"])
+        def self.chain
+          @chain ||= ::Sidekiq::Middleware::Chain.new do |chain|
+            chain.add Labkit::Middleware::Sidekiq::Context::Server
+            chain.add Labkit::Middleware::Sidekiq::Tracing::Server if Labkit::Tracing.enabled?
+          end
+        end
 
-          Labkit::Context.with_context(data) do |_context|
+        def call(*args)
+          self.class.chain.invoke(*args) do
             yield
           end
         end

data/lib/labkit/middleware/sidekiq/tracing.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Middleware
+    module Sidekiq
+      # Sidekiq provides classes for instrumenting Sidekiq client and server
+      # functionality
+      module Tracing
+        autoload :Client, "labkit/middleware/sidekiq/tracing/client"
+        autoload :Server, "labkit/middleware/sidekiq/tracing/server"
+        autoload :SidekiqCommon, "labkit/middleware/sidekiq/tracing/sidekiq_common"
+      end
+    end
+  end
+end

data/lib/labkit/middleware/sidekiq/tracing/client.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "opentracing"
+
+module Labkit
+  module Middleware
+    module Sidekiq
+      module Tracing
+        # Client provides a sidekiq client middleware for
+        # instrumenting distributed tracing calls made from the client
+        # application
+        class Client
+          include SidekiqCommon
+
+          SPAN_KIND = "client"
+
+          def call(_worker_class, job, _queue, _redis_pool)
+            Labkit::Tracing::TracingUtils.with_tracing(operation_name: "sidekiq:#{job["class"]}", tags: tags_from_job(job, SPAN_KIND)) do |span|
+              # Inject the details directly into the job
+              Labkit::Tracing::TracingUtils.tracer.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, job)
+
+              yield
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/middleware/sidekiq/tracing/server.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "opentracing"
+
+module Labkit
+  module Middleware
+    module Sidekiq
+      module Tracing
+        # Server provides a sidekiq server middleware for
+        # instrumenting distributed tracing calls when they are
+        # executed by the Sidekiq server
+        class Server
+          include SidekiqCommon
+
+          SPAN_KIND = "server"
+
+          def call(_worker, job, _queue)
+            context = Labkit::Tracing::TracingUtils.tracer.extract(OpenTracing::FORMAT_TEXT_MAP, job)
+
+            Labkit::Tracing::TracingUtils.with_tracing(operation_name: "sidekiq:#{job["class"]}", child_of: context, tags: tags_from_job(job, SPAN_KIND)) { |_span| yield }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/middleware/sidekiq/tracing/sidekiq_common.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Middleware
+    module Sidekiq
+      module Tracing
+        # SidekiqCommon is a mixin for the sidekiq middleware components
+        module SidekiqCommon
+          def tags_from_job(job, kind)
+            {
+              "component" => "sidekiq",
+              "span.kind" => kind,
+              "sidekiq.queue" => job["queue"],
+              "sidekiq.jid" => job["jid"],
+              "sidekiq.retry" => job["retry"].to_s,
+              "sidekiq.args" => job["args"]&.join(", "),
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing.rb

@@ -28,6 +28,10 @@ module Labkit
       ENV["GITLAB_TRACING_URL"]
     end
 
+    def self.stacktrace_operations
+      @stacktrace_operations ||= Set.new(ENV["GITLAB_TRACING_INCLUDE_STACKTRACE"].to_s.split(",").map(&:strip))
+    end
+
     def self.tracing_url_enabled?
       enabled? && tracing_url_template.present?
     end

data/lib/labkit/tracing/rails.rb

@@ -4,10 +4,15 @@ module Labkit
   module Tracing
     # Rails provides classes for instrumenting Rails events
     module Rails
-      autoload :ActionViewSubscriber, "labkit/tracing/rails/action_view_subscriber"
-      autoload :ActiveRecordSubscriber, "labkit/tracing/rails/active_record_subscriber"
-      autoload :ActiveSupportSubscriber, "labkit/tracing/rails/active_support_subscriber"
+      autoload :AbstractInstrumenter, "labkit/tracing/rails/abstract_instrumenter"
+      autoload :ActionView, "labkit/tracing/rails/action_view"
+      autoload :ActiveRecord, "labkit/tracing/rails/active_record"
+      autoload :ActiveSupport, "labkit/tracing/rails/active_support"
       autoload :RailsCommon, "labkit/tracing/rails/rails_common"
+
+      ActionViewSubscriber = ActionView::Subscriber
+      ActiveRecordSubscriber = ActiveRecord::Subscriber
+      ActiveSupportSubscriber = ActiveSupport::Subscriber
     end
   end
 end

data/lib/labkit/tracing/rails/abstract_instrumenter.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "active_support/all"
+
+module Labkit
+  module Tracing
+    module Rails
+      # https://edgeapi.rubyonrails.org/classes/ActiveSupport/Notifications/Instrumenter.html#method-c-new
+      class AbstractInstrumenter
+        def start(name, id, payload)
+          scope = OpenTracing.start_active_span(span_name(payload))
+
+          scope_stack.push scope
+        end
+
+        def finish(name, id, payload)
+          scope = scope_stack.pop
+          span = scope.span
+
+          Labkit::Tracing::TracingUtils.log_common_fields_on_span(span, span_name(payload))
+
+          exception = payload[:exception]
+          Labkit::Tracing::TracingUtils.log_exception_on_span(span, exception) if exception
+
+          tags(payload).each do |k, v|
+            span.set_tag(k, v)
+          end
+
+          scope.close
+        end
+
+        def scope_stack
+          Thread.current[:_labkit_trace_scope_stack] ||= []
+        end
+
+        def span_name(payload)
+          raise "span_name not implemented"
+        end
+
+        def tags(payload)
+          {}
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/action_view.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActionView
+        autoload :RenderCollectionInstrumenter, "labkit/tracing/rails/action_view/render_collection_instrumenter"
+        autoload :RenderPartialInstrumenter, "labkit/tracing/rails/action_view/render_partial_instrumenter"
+        autoload :RenderTemplateInstrumenter, "labkit/tracing/rails/action_view/render_template_instrumenter"
+        autoload :Subscriber, "labkit/tracing/rails/action_view/subscriber"
+
+        COMPONENT_TAG = "ActionView"
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/action_view/render_collection_instrumenter.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActionView
+        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
+        class RenderCollectionInstrumenter < AbstractInstrumenter
+          def span_name(payload)
+            "render_collection"
+          end
+
+          def tags(payload)
+            {
+              "component" => COMPONENT_TAG,
+              "template.id" => payload[:identifier],
+              "template.count" => payload[:count] || 0,
+              "template.cache.hits" => payload[:cache_hits] || 0,
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/action_view/render_partial_instrumenter.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActionView
+        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
+        class RenderPartialInstrumenter < AbstractInstrumenter
+          def span_name(payload)
+            "render_partial"
+          end
+
+          def tags(payload)
+            { "component" => COMPONENT_TAG, "template.id" => payload[:identifier] }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/action_view/render_template_instrumenter.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActionView
+        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
+        class RenderTemplateInstrumenter < AbstractInstrumenter
+          def span_name(payload)
+            "render_template"
+          end
+
+          def tags(payload)
+            { "component" => COMPONENT_TAG, "template.id" => payload[:identifier], "template.layout" => payload[:layout] }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/action_view/subscriber.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActionView
+        # ActionView bridges action view notifications to
+        # the distributed tracing subsystem
+        class Subscriber
+          include RailsCommon
+
+          RENDER_TEMPLATE_NOTIFICATION_TOPIC = "render_template.action_view"
+          RENDER_COLLECTION_NOTIFICATION_TOPIC = "render_collection.action_view"
+          RENDER_PARTIAL_NOTIFICATION_TOPIC = "render_partial.action_view"
+
+          # Instruments Rails ActionView events for opentracing.
+          # Returns a lambda, which, when called will unsubscribe from the notifications
+          def self.instrument
+            subscriptions = [
+              ::ActiveSupport::Notifications.subscribe(RENDER_TEMPLATE_NOTIFICATION_TOPIC, RenderTemplateInstrumenter.new),
+              ::ActiveSupport::Notifications.subscribe(RENDER_COLLECTION_NOTIFICATION_TOPIC, RenderCollectionInstrumenter.new),
+              ::ActiveSupport::Notifications.subscribe(RENDER_PARTIAL_NOTIFICATION_TOPIC, RenderPartialInstrumenter.new),
+            ]
+
+            create_unsubscriber subscriptions
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/active_record.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActiveRecord
+        autoload :SqlInstrumenter, "labkit/tracing/rails/active_record/sql_instrumenter"
+        autoload :Subscriber, "labkit/tracing/rails/active_record/subscriber"
+
+        COMPONENT_TAG = "ActiveRecord"
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/active_record/sql_instrumenter.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActiveRecord
+        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
+        class SqlInstrumenter < AbstractInstrumenter
+          OPERATION_NAME_PREFIX = "active_record:"
+          DEFAULT_OPERATION_NAME = "sqlquery"
+
+          def span_name(payload)
+            OPERATION_NAME_PREFIX + (payload[:name].presence || DEFAULT_OPERATION_NAME)
+          end
+
+          def tags(payload)
+            {
+              "component" => COMPONENT_TAG,
+              "span.kind" => "client",
+              "db.type" => "sql",
+              "db.connection_id" => payload[:connection_id],
+              "db.cached" => payload[:cached] || false,
+              "db.statement" => payload[:sql],
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/active_record/subscriber.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActiveRecord
+        # ActiveRecord bridges active record notifications to
+        # the distributed tracing subsystem
+        class Subscriber
+          include RailsCommon
+
+          ACTIVE_RECORD_NOTIFICATION_TOPIC = "sql.active_record"
+
+          # Instruments Rails ActiveRecord events for opentracing.
+          # Returns a lambda, which, when called will unsubscribe from the notifications
+          def self.instrument
+            subscription = ::ActiveSupport::Notifications.subscribe(ACTIVE_RECORD_NOTIFICATION_TOPIC, SqlInstrumenter.new)
+
+            create_unsubscriber [subscription]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/active_support.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActiveSupport
+        autoload :CacheDeleteInstrumenter, "labkit/tracing/rails/active_support/cache_delete_instrumenter"
+        autoload :CacheFetchHitInstrumenter, "labkit/tracing/rails/active_support/cache_fetch_hit_instrumenter"
+        autoload :CacheGenerateInstrumenter, "labkit/tracing/rails/active_support/cache_generate_instrumenter"
+        autoload :CacheReadInstrumenter, "labkit/tracing/rails/active_support/cache_read_instrumenter"
+        autoload :CacheWriteInstrumenter, "labkit/tracing/rails/active_support/cache_write_instrumenter"
+        autoload :Subscriber, "labkit/tracing/rails/active_support/subscriber"
+
+        COMPONENT_TAG = "ActiveSupport"
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/active_support/cache_delete_instrumenter.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActiveSupport
+        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
+        class CacheDeleteInstrumenter < AbstractInstrumenter
+          def span_name(payload)
+            "cache_delete"
+          end
+
+          def tags(payload)
+            { "component" => COMPONENT_TAG, "cache.key" => payload[:key] }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/active_support/cache_fetch_hit_instrumenter.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActiveSupport
+        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
+        class CacheFetchHitInstrumenter < AbstractInstrumenter
+          def span_name(payload)
+            "cache_fetch_hit"
+          end
+
+          def tags(payload)
+            { "component" => COMPONENT_TAG, "cache.key" => payload[:key] }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/active_support/cache_generate_instrumenter.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActiveSupport
+        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
+        class CacheGenerateInstrumenter < AbstractInstrumenter
+          def span_name(payload)
+            "cache_generate"
+          end
+
+          def tags(payload)
+            { "component" => COMPONENT_TAG, "cache.key" => payload[:key] }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/active_support/cache_read_instrumenter.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActiveSupport
+        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
+        class CacheReadInstrumenter < AbstractInstrumenter
+          def span_name(payload)
+            "cache_read"
+          end
+
+          def tags(payload)
+            {
+              "component" => COMPONENT_TAG,
+              "cache.key" => payload[:key],
+              "cache.hit" => payload[:hit],
+              "cache.super_operation" => payload[:super_operation],
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/active_support/cache_write_instrumenter.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActiveSupport
+        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
+        class CacheWriteInstrumenter < AbstractInstrumenter
+          def span_name(payload)
+            "cache_write"
+          end
+
+          def tags(payload)
+            { "component" => COMPONENT_TAG, "cache.key" => payload[:key] }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/active_support/subscriber.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module Rails
+      module ActiveSupport
+        # ActiveSupport bridges action active support notifications to
+        # the distributed tracing subsystem
+        class Subscriber
+          include RailsCommon
+
+          CACHE_READ_TOPIC = "cache_read.active_support"
+          CACHE_GENERATE_TOPIC = "cache_generate.active_support"
+          CACHE_FETCH_HIT_TOPIC = "cache_fetch_hit.active_support"
+          CACHE_WRITE_TOPIC = "cache_write.active_support"
+          CACHE_DELETE_TOPIC = "cache_delete.active_support"
+
+          # Instruments Rails ActiveSupport events for opentracing.
+          # Returns a lambda, which, when called will unsubscribe from the notifications
+          def self.instrument
+            subscriptions = [
+              ::ActiveSupport::Notifications.subscribe(CACHE_READ_TOPIC, CacheReadInstrumenter.new),
+              ::ActiveSupport::Notifications.subscribe(CACHE_GENERATE_TOPIC, CacheGenerateInstrumenter.new),
+              ::ActiveSupport::Notifications.subscribe(CACHE_FETCH_HIT_TOPIC, CacheFetchHitInstrumenter.new),
+              ::ActiveSupport::Notifications.subscribe(CACHE_WRITE_TOPIC, CacheWriteInstrumenter.new),
+              ::ActiveSupport::Notifications.subscribe(CACHE_DELETE_TOPIC, CacheDeleteInstrumenter.new),
+            ]
+
+            create_unsubscriber subscriptions
+          end
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails/rails_common.rb

@@ -8,19 +8,13 @@ module Labkit
       # RailsCommon is a mixin for providing instrumentation
       # functionality for the rails instrumentation classes
       module RailsCommon
-        extend ActiveSupport::Concern
+        extend ::ActiveSupport::Concern
 
         class_methods do
           def create_unsubscriber(subscriptions)
-            -> { subscriptions.each { |subscriber| ActiveSupport::Notifications.unsubscribe(subscriber) } }
+            -> { subscriptions.each { |subscriber| ::ActiveSupport::Notifications.unsubscribe(subscriber) } }
           end
         end
-
-        def generate_span_for_notification(operation_name, start, finish, payload, tags)
-          exception = payload[:exception]
-
-          TracingUtils.postnotify_span(operation_name, start, finish, tags: tags, exception: exception)
-        end
       end
     end
   end

data/lib/labkit/tracing/redis/redis_interceptor_helper.rb

@@ -8,7 +8,7 @@ module Labkit
       # RedisInterceptorHelper is a helper for the RedisInterceptor. This is not a public API
       class RedisInterceptorHelper
         # For optimization, compile this once
-        MASK_REDIS_RE = /^([\w-]+(?:\W+[\w-]+(?:\W+[\w-]+)?)?)(.?)/.freeze
+        MASK_REDIS_RE = /^([\w{}-]+(?:\W+[\w{}-]+(?:\W+[\w{}-]+)?)?)(.?)/.freeze
 
         def self.call_with_tracing(command, client)
           Labkit::Tracing::TracingUtils.with_tracing(operation_name: "redis.call", tags: tags_from_command(command, client)) do |_span|
@@ -99,6 +99,7 @@ module Labkit
           return "" if argument.empty?
 
           matches = argument.match(MASK_REDIS_RE)
+
           matches[2].empty? ? matches[0] : matches[0] + "*****"
         end
         private_class_method :mask_redis_arg

data/lib/labkit/tracing/tracing_utils.rb

@@ -12,9 +12,7 @@ module Labkit
         scope = tracer.start_active_span(operation_name, child_of: child_of, tags: tags)
         span = scope.span
 
-        # Add correlation details to the span if we have them
-        correlation_id = Labkit::Correlation::CorrelationId.current_id
-        span.set_tag("correlation_id", correlation_id) if correlation_id
+        log_common_fields_on_span(span, operation_name)
 
         begin
           yield span
@@ -35,11 +33,19 @@ module Labkit
       def self.postnotify_span(operation_name, start_time, end_time, tags: nil, child_of: nil, exception: nil)
         span = OpenTracing.start_span(operation_name, start_time: start_time, tags: tags, child_of: child_of)
 
+        log_common_fields_on_span(span, operation_name)
         log_exception_on_span(span, exception) if exception
 
         span.finish(end_time: end_time)
       end
 
+      # Add common fields to a span
+      def self.log_common_fields_on_span(span, operation_name)
+        correlation_id = Labkit::Correlation::CorrelationId.current_id
+        span.set_tag("correlation_id", correlation_id) if correlation_id
+        span.log_kv(stack: caller.join('\n')) if include_stacktrace?(operation_name)
+      end
+
       # Add exception logging to a span
       def self.log_exception_on_span(span, exception)
         span.set_tag("error", true)
@@ -60,6 +66,14 @@ module Labkit
           { :"event" => "error", :"error.kind" => exception.class.to_s, :"error.object" => Labkit::Logging::Sanitizer.sanitize_field(exception.to_s) }
         end
       end
+
+      def self.include_stacktrace?(operation_name)
+        @include_stacktrace ||= Hash.new do |result, name|
+          result[name] = Tracing.stacktrace_operations.any? { |stacktrace_operation| name.starts_with?(stacktrace_operation) }
+        end
+
+        @include_stacktrace[operation_name]
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-labkit
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.12.1
 platform: ruby
 authors:
 - Andrew Newdigate
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-20 00:00:00.000000000 Z
+date: 2020-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -284,7 +284,14 @@ files:
 - lib/labkit/middleware/rack.rb
 - lib/labkit/middleware/sidekiq.rb
 - lib/labkit/middleware/sidekiq/client.rb
+- lib/labkit/middleware/sidekiq/context.rb
+- lib/labkit/middleware/sidekiq/context/client.rb
+- lib/labkit/middleware/sidekiq/context/server.rb
 - lib/labkit/middleware/sidekiq/server.rb
+- lib/labkit/middleware/sidekiq/tracing.rb
+- lib/labkit/middleware/sidekiq/tracing/client.rb
+- lib/labkit/middleware/sidekiq/tracing/server.rb
+- lib/labkit/middleware/sidekiq/tracing/sidekiq_common.rb
 - lib/labkit/tracing.rb
 - lib/labkit/tracing/factory.rb
 - lib/labkit/tracing/grpc.rb
@@ -294,17 +301,26 @@ files:
 - lib/labkit/tracing/jaeger_factory.rb
 - lib/labkit/tracing/rack_middleware.rb
 - lib/labkit/tracing/rails.rb
-- lib/labkit/tracing/rails/action_view_subscriber.rb
-- lib/labkit/tracing/rails/active_record_subscriber.rb
-- lib/labkit/tracing/rails/active_support_subscriber.rb
+- lib/labkit/tracing/rails/abstract_instrumenter.rb
+- lib/labkit/tracing/rails/action_view.rb
+- lib/labkit/tracing/rails/action_view/render_collection_instrumenter.rb
+- lib/labkit/tracing/rails/action_view/render_partial_instrumenter.rb
+- lib/labkit/tracing/rails/action_view/render_template_instrumenter.rb
+- lib/labkit/tracing/rails/action_view/subscriber.rb
+- lib/labkit/tracing/rails/active_record.rb
+- lib/labkit/tracing/rails/active_record/sql_instrumenter.rb
+- lib/labkit/tracing/rails/active_record/subscriber.rb
+- lib/labkit/tracing/rails/active_support.rb
+- lib/labkit/tracing/rails/active_support/cache_delete_instrumenter.rb
+- lib/labkit/tracing/rails/active_support/cache_fetch_hit_instrumenter.rb
+- lib/labkit/tracing/rails/active_support/cache_generate_instrumenter.rb
+- lib/labkit/tracing/rails/active_support/cache_read_instrumenter.rb
+- lib/labkit/tracing/rails/active_support/cache_write_instrumenter.rb
+- lib/labkit/tracing/rails/active_support/subscriber.rb
 - lib/labkit/tracing/rails/rails_common.rb
 - lib/labkit/tracing/redis.rb
 - lib/labkit/tracing/redis/redis_interceptor.rb
 - lib/labkit/tracing/redis/redis_interceptor_helper.rb
-- lib/labkit/tracing/sidekiq.rb
-- lib/labkit/tracing/sidekiq/client_middleware.rb
-- lib/labkit/tracing/sidekiq/server_middleware.rb
-- lib/labkit/tracing/sidekiq/sidekiq_common.rb
 - lib/labkit/tracing/tracing_utils.rb
 homepage: https://gitlab.com/gitlab-org/labkit-ruby
 licenses:

data/lib/labkit/tracing/rails/action_view_subscriber.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-module Labkit
-  module Tracing
-    module Rails
-      # ActionViewSubscriber bridges action view notifications to
-      # the distributed tracing subsystem
-      class ActionViewSubscriber
-        include RailsCommon
-
-        COMPONENT_TAG = "ActionView"
-        RENDER_TEMPLATE_NOTIFICATION_TOPIC = "render_template.action_view"
-        RENDER_COLLECTION_NOTIFICATION_TOPIC = "render_collection.action_view"
-        RENDER_PARTIAL_NOTIFICATION_TOPIC = "render_partial.action_view"
-
-        # Instruments Rails ActionView events for opentracing.
-        # Returns a lambda, which, when called will unsubscribe from the notifications
-        def self.instrument
-          subscriber = new
-
-          subscriptions = [
-            ActiveSupport::Notifications.subscribe(RENDER_TEMPLATE_NOTIFICATION_TOPIC) do |_, start, finish, _, payload|
-              subscriber.notify_render_template(start, finish, payload)
-            end,
-            ActiveSupport::Notifications.subscribe(RENDER_COLLECTION_NOTIFICATION_TOPIC) do |_, start, finish, _, payload|
-              subscriber.notify_render_collection(start, finish, payload)
-            end,
-            ActiveSupport::Notifications.subscribe(RENDER_PARTIAL_NOTIFICATION_TOPIC) do |_, start, finish, _, payload|
-              subscriber.notify_render_partial(start, finish, payload)
-            end,
-          ]
-
-          create_unsubscriber subscriptions
-        end
-
-        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
-        def notify_render_template(start, finish, payload)
-          generate_span_for_notification("render_template", start, finish, payload, tags_for_render_template(payload))
-        end
-
-        def notify_render_collection(start, finish, payload)
-          generate_span_for_notification("render_collection", start, finish, payload, tags_for_render_collection(payload))
-        end
-
-        def notify_render_partial(start, finish, payload)
-          generate_span_for_notification("render_partial", start, finish, payload, tags_for_render_partial(payload))
-        end
-
-        private
-
-        def tags_for_render_template(payload)
-          { "component" => COMPONENT_TAG, "template.id" => payload[:identifier], "template.layout" => payload[:layout] }
-        end
-
-        def tags_for_render_collection(payload)
-          {
-            "component" => COMPONENT_TAG,
-            "template.id" => payload[:identifier],
-            "template.count" => payload[:count] || 0,
-            "template.cache.hits" => payload[:cache_hits] || 0,
-          }
-        end
-
-        def tags_for_render_partial(payload)
-          { "component" => COMPONENT_TAG, "template.id" => payload[:identifier] }
-        end
-      end
-    end
-  end
-end

data/lib/labkit/tracing/rails/active_record_subscriber.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module Labkit
-  module Tracing
-    module Rails
-      # ActiveRecordSubscriber bridges active record notifications to
-      # the distributed tracing subsystem
-      class ActiveRecordSubscriber
-        include RailsCommon
-
-        ACTIVE_RECORD_NOTIFICATION_TOPIC = "sql.active_record"
-        OPERATION_NAME_PREFIX = "active_record:"
-        DEFAULT_OPERATION_NAME = "sqlquery"
-
-        # Instruments Rails ActiveRecord events for opentracing.
-        # Returns a lambda, which, when called will unsubscribe from the notifications
-        def self.instrument
-          subscriber = new
-
-          subscription =
-            ActiveSupport::Notifications.subscribe(ACTIVE_RECORD_NOTIFICATION_TOPIC) do |_, start, finish, _, payload|
-              subscriber.notify(start, finish, payload)
-            end
-
-          create_unsubscriber [subscription]
-        end
-
-        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
-        def notify(start, finish, payload)
-          generate_span_for_notification(notification_name(payload), start, finish, payload, tags_for_notification(payload))
-        end
-
-        private
-
-        def notification_name(payload)
-          OPERATION_NAME_PREFIX + (payload[:name].presence || DEFAULT_OPERATION_NAME)
-        end
-
-        def tags_for_notification(payload)
-          {
-            "component" => "ActiveRecord",
-            "span.kind" => "client",
-            "db.type" => "sql",
-            "db.connection_id" => payload[:connection_id],
-            "db.cached" => payload[:cached] || false,
-            "db.statement" => payload[:sql],
-          }
-        end
-      end
-    end
-  end
-end

data/lib/labkit/tracing/rails/active_support_subscriber.rb

@@ -1,86 +0,0 @@
-# frozen_string_literal: true
-
-module Labkit
-  module Tracing
-    module Rails
-      # ActiveSupport bridges action active support notifications to
-      # the distributed tracing subsystem
-      class ActiveSupportSubscriber
-        include RailsCommon
-
-        COMPONENT_TAG = "ActiveSupport"
-
-        CACHE_READ_TOPIC = "cache_read.active_support"
-        CACHE_GENERATE_TOPIC = "cache_generate.active_support"
-        CACHE_FETCH_HIT_TOPIC = "cache_fetch_hit.active_support"
-        CACHE_WRITE_TOPIC = "cache_write.active_support"
-        CACHE_DELETE_TOPIC = "cache_delete.active_support"
-
-        # Instruments Rails ActiveSupport events for opentracing.
-        # Returns a lambda, which, when called will unsubscribe from the notifications
-        def self.instrument
-          subscriber = new
-
-          subscriptions = [
-            ActiveSupport::Notifications.subscribe(CACHE_READ_TOPIC) do |_, start, finish, _, payload|
-              subscriber.notify_cache_read(start, finish, payload)
-            end,
-            ActiveSupport::Notifications.subscribe(CACHE_GENERATE_TOPIC) do |_, start, finish, _, payload|
-              subscriber.notify_cache_generate(start, finish, payload)
-            end,
-            ActiveSupport::Notifications.subscribe(CACHE_FETCH_HIT_TOPIC) do |_, start, finish, _, payload|
-              subscriber.notify_cache_fetch_hit(start, finish, payload)
-            end,
-            ActiveSupport::Notifications.subscribe(CACHE_WRITE_TOPIC) do |_, start, finish, _, payload|
-              subscriber.notify_cache_write(start, finish, payload)
-            end,
-            ActiveSupport::Notifications.subscribe(CACHE_DELETE_TOPIC) do |_, start, finish, _, payload|
-              subscriber.notify_cache_delete(start, finish, payload)
-            end,
-          ]
-
-          create_unsubscriber subscriptions
-        end
-
-        # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html#active-support
-        def notify_cache_read(start, finish, payload)
-          generate_span_for_notification("cache_read", start, finish, payload, tags_for_cache_read(payload))
-        end
-
-        def notify_cache_generate(start, finish, payload)
-          generate_span_for_notification("cache_generate", start, finish, payload, tags_for_key(payload))
-        end
-
-        def notify_cache_fetch_hit(start, finish, payload)
-          generate_span_for_notification("cache_fetch_hit", start, finish, payload, tags_for_key(payload))
-        end
-
-        def notify_cache_write(start, finish, payload)
-          generate_span_for_notification("cache_write", start, finish, payload, tags_for_key(payload))
-        end
-
-        def notify_cache_delete(start, finish, payload)
-          generate_span_for_notification("cache_delete", start, finish, payload, tags_for_key(payload))
-        end
-
-        private
-
-        def tags_for_cache_read(payload)
-          {
-            "component" => COMPONENT_TAG,
-            "cache.key" => payload[:key],
-            "cache.hit" => payload[:hit],
-            "cache.super_operation" => payload[:super_operation],
-          }
-        end
-
-        def tags_for_key(payload)
-          {
-            "component" => COMPONENT_TAG,
-            "cache.key" => payload[:key],
-          }
-        end
-      end
-    end
-  end
-end

data/lib/labkit/tracing/sidekiq.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Labkit
-  module Tracing
-    # Sidekiq provides classes for instrumenting Sidekiq client and server
-    # functionality
-    module Sidekiq
-      autoload :ClientMiddleware, "labkit/tracing/sidekiq/client_middleware"
-      autoload :ServerMiddleware, "labkit/tracing/sidekiq/server_middleware"
-      autoload :SidekiqCommon, "labkit/tracing/sidekiq/sidekiq_common"
-    end
-  end
-end

data/lib/labkit/tracing/sidekiq/client_middleware.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-require "opentracing"
-
-module Labkit
-  module Tracing
-    module Sidekiq
-      # ClientMiddleware provides a sidekiq client middleware for
-      # instrumenting distributed tracing calls made from the client
-      # application
-      class ClientMiddleware
-        include SidekiqCommon
-
-        SPAN_KIND = "client"
-
-        def call(_worker_class, job, _queue, _redis_pool)
-          TracingUtils.with_tracing(operation_name: "sidekiq:#{job["class"]}", tags: tags_from_job(job, SPAN_KIND)) do |span|
-            # Inject the details directly into the job
-            TracingUtils.tracer.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, job)
-
-            yield
-          end
-        end
-      end
-    end
-  end
-end

data/lib/labkit/tracing/sidekiq/server_middleware.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-require "opentracing"
-
-module Labkit
-  module Tracing
-    module Sidekiq
-      # ServerMiddleware provides a sidekiq server middleware for
-      # instrumenting distributed tracing calls when they are
-      # executed by the Sidekiq server
-      class ServerMiddleware
-        include SidekiqCommon
-
-        SPAN_KIND = "server"
-
-        def call(_worker, job, _queue)
-          context = TracingUtils.tracer.extract(OpenTracing::FORMAT_TEXT_MAP, job)
-
-          TracingUtils.with_tracing(operation_name: "sidekiq:#{job["class"]}", child_of: context, tags: tags_from_job(job, SPAN_KIND)) { |_span| yield }
-        end
-      end
-    end
-  end
-end

data/lib/labkit/tracing/sidekiq/sidekiq_common.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Labkit
-  module Tracing
-    module Sidekiq
-      # SidekiqCommon is a mixin for the sidekiq middleware components
-      module SidekiqCommon
-        def tags_from_job(job, kind)
-          {
-            "component" => "sidekiq",
-            "span.kind" => kind,
-            "sidekiq.queue" => job["queue"],
-            "sidekiq.jid" => job["jid"],
-            "sidekiq.retry" => job["retry"].to_s,
-            "sidekiq.args" => job["args"]&.join(", "),
-          }
-        end
-      end
-    end
-  end
-end