gitlab-labkit 0.13.5 → 0.16.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 386875226d2cd983130f209da6204b633bc564dc7cdf749e4fd51aeb27ea44b7
-  data.tar.gz: b07771c72a4dc8385bba052ac9524dde00cce55023ad9c327db040535faeed31
+  metadata.gz: db0f8ac56d95729fa2008c5b65bf1630137b99e977e09d1b5090b7a890c8f9a3
+  data.tar.gz: 94c467fca0a2d54afde08588b34e01a46c3b0c62db8384cd84db125504949413
 SHA512:
-  metadata.gz: 73dcbe4d30d4706da4299dce292b7cd7271be8c10ab555fa4bbd46c53f3cdbb2d47e1c92743f4f914a95b78d3035d43e38003344f7afc399b5507dab84690bb1
-  data.tar.gz: 571c2df46e971b9696c5e23b5bdba9ab6df56d8253e998fa9a8797b2ebfb4d0e1b2371ac1626fd8e02d11641b17fb5270e23a7c927e28fddf03b17f4b3b21e7a
+  metadata.gz: 9617b98e2e29783bb2b644da067283e3166e1b78298d34f853431a94a6f4da7a1f219315697d958d0b3ed66dae8eceb4b8a1df409397999c09b3b6030e288f9d
+  data.tar.gz: 2dfc9f64306996c329795db9e5c463dfed0814fa1eb6430cd7bb788a0bc4ba757c70c9daeaf118f5d5efedbe829e4b393fffa2b95ebc88b035cbb2d58fee7633

data/.gitlab/CODEOWNERS

@@ -0,0 +1 @@
+* @andrewn @ayufan @reprazent

data/.rubocop.yml

@@ -28,6 +28,9 @@ Style/StringLiterals:
 Style/StringLiteralsInInterpolation:
   EnforcedStyle: double_quotes
 
+Style/UseLambda:
+  Enabled: false
+
 Layout/MultilineMethodCallIndentation:
   Enabled: No
 

data/gitlab-labkit.gemspec

@@ -19,19 +19,24 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = ">= 2.4.0"
 
   # Please maintain alphabetical order for dependencies
-  spec.add_runtime_dependency "actionpack", ">= 5.0.0", "< 6.1.0"
-  spec.add_runtime_dependency "activesupport", ">= 5.0.0", "< 6.1.0"
+  spec.add_runtime_dependency "actionpack", ">= 5.0.0", "< 7.0.0"
+  spec.add_runtime_dependency "activesupport", ">= 5.0.0", "< 7.0.0"
   spec.add_runtime_dependency "grpc", "~> 1.19" # Be sure to update the "grpc-tools" dev_depenency too
   spec.add_runtime_dependency "jaeger-client", "~> 1.1"
   spec.add_runtime_dependency "opentracing", "~> 0.4"
+  spec.add_runtime_dependency "pg_query", "~> 1.3"
   spec.add_runtime_dependency "redis", ">3.0.0", "<5.0.0"
-  spec.add_runtime_dependency "gitlab-pg_query", "~> 1.3"
 
   # Please maintain alphabetical order for dev dependencies
+  spec.add_development_dependency "excon", "~> 0.78.1"
+  spec.add_development_dependency "faraday", "~> 1.2.0"
   spec.add_development_dependency "grpc-tools", "~> 1.19"
+  spec.add_development_dependency "httparty", "~> 0.17.3"
+  spec.add_development_dependency "httpclient", "~> 2.8.3"
   spec.add_development_dependency "pry", "~> 0.12"
   spec.add_development_dependency "rack", "~> 2.0"
   spec.add_development_dependency "rake", "~> 12.3"
+  spec.add_development_dependency "rest-client", "~> 2.1.0"
   spec.add_development_dependency "rspec", "~> 3.8.0"
   spec.add_development_dependency "rspec-parameterized", "~> 0.4"
   spec.add_development_dependency "rubocop", "~> 0.65.0"

data/lib/gitlab-labkit.rb

@@ -7,11 +7,40 @@ require "active_support/all"
 # infrastructural concerns, partcularly related to
 # observability.
 module Labkit
+  autoload :System, "labkit/system"
+
   autoload :Correlation, "labkit/correlation"
   autoload :Context, "labkit/context"
   autoload :Tracing, "labkit/tracing"
   autoload :Logging, "labkit/logging"
   autoload :Middleware, "labkit/middleware"
+
+  # Publishers to publish notifications whenever a HTTP reqeust is made.
+  # A broadcasted notification's payload in topic "request.external_http" includes:
+  #   + method (String): "GET"
+  #   + code (String): "200" # This is the status code read directly from HTTP response
+  #   + duration (Float - seconds): 0.234
+  #   + host (String): "gitlab.com"
+  #   + port (Integer): 80,
+  #   + path (String): "/gitlab-org/gitlab"
+  #   + scheme (String): "https"
+  #   + query (String): "field_a=1&field_b=2"
+  #   + fragment (String): "issue-number-1"
+  #   + proxy_host (String - Optional): "proxy.gitlab.com"
+  #   + proxy_port (Integer - Optional): 80
+  #   + exception (Array<String> - Optional): ["Net::ReadTimeout", "Net::ReadTimeout with #<TCPSocket:(closed)>"]
+  #   + exception_object (Error Object - Optional): #<Net::ReadTimeout: Net::ReadTimeout>
+  #
+  # Usage:
+  #
+  # ActiveSupport::Notifications.subscribe "request.external_http" do |name, started, finished, unique_id, data|
+  #   puts "#{name} | #{started} | #{finished} | #{unique_id} | #{data.inspect}"
+  # end
+  #
+  EXTERNAL_HTTP_NOTIFICATION_TOPIC = "request.external_http"
+  autoload :NetHttpPublisher, "labkit/net_http_publisher"
+  autoload :ExconPublisher, "labkit/excon_publisher"
+  autoload :HTTPClientPublisher, "labkit/httpclient_publisher"
 end
 
 # rubocop:enable Naming/FileName

data/lib/labkit/context.rb

@@ -23,7 +23,7 @@ module Labkit
     RAW_KEYS = [CORRELATION_ID_KEY].freeze
     HEADER_PREFIX = "X-Gitlab-"
     KNOWN_KEYS = %w[user project root_namespace subscription_plan caller_id
-                    remote_ip related_class feature_category].freeze
+                    remote_ip related_class feature_category client_id].freeze
 
     class << self
       def with_context(attributes = {})
@@ -106,6 +106,12 @@ module Labkit
       end
     end
 
+    def get_attribute(attribute)
+      raw = call_or_value(data[log_key(attribute)])
+
+      call_or_value(raw)
+    end
+
     protected
 
     def assign_attributes(attributes)
@@ -130,9 +136,13 @@ module Labkit
 
     attr_reader :data
 
+    def call_or_value(value)
+      value.respond_to?(:call) ? value.call : value
+    end
+
     def expand_data
       data.transform_values do |value|
-        value = value.respond_to?(:call) ? value.call : value
+        value = call_or_value(value)
 
         value.presence
       end.compact

data/lib/labkit/excon_publisher.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module Labkit
+  ##
+  # A middleware for Excon HTTP library to publish a notification
+  # whenever a HTTP request is triggered.
+  #
+  # Excon supports a middleware system that allows request/response
+  # interception freely. Whenever a new Excon connection is created, a list of
+  # default middlewares is injected. This list of middlewares can be altered
+  # thanks to Excon.defaults accessor. ExconPublisher is inserted into this
+  # list. It affects all connections created in future. There is a limitation
+  # that this approach doesn't work if a user decides to override the default
+  # middleware list. It is unlikely though, at least in the dependency tree of
+  # GitLab.
+  #
+  # ExconPublisher instance is created once and shared between all Excon
+  # connections later. Each connection may be triggered by different threads in
+  # parallel. In such cases, a connection objects creates multiple sockets for
+  # each thread. Therfore in the implementation of this middleware, the
+  # instrumation payload for each connection is stored inside a thread-isolated
+  # storage.
+  #
+  # For more information:
+  # https://github.com/excon/excon/blob/81a0130537f2f8cd00d6daafb05d02d9a90dc9f7/lib/excon/middlewares/base.rb
+  # https://github.com/excon/excon/blob/fa3ec51e9bb062a12846a1cfff09534e76c99f4b/lib/excon/constants.rb#L146
+  # https://github.com/excon/excon/blob/fa3ec51e9bb062a12846a1cfff09534e76c99f4b/lib/excon/connection.rb#L474
+  class ExconPublisher
+    @prepend_mutex = Mutex.new
+
+    def self.labkit_prepend!
+      @prepend_mutex.synchronize do
+        return if !defined?(Excon) || @prepended
+
+        defaults = Excon.defaults
+        defaults[:middlewares] << ExconPublisher
+
+        @prepended = true
+      end
+    end
+
+    def initialize(stack)
+      @stack = stack
+      @instrumenter = ActiveSupport::Notifications.instrumenter
+    end
+
+    def request_call(datum)
+      payload = start_payload(datum)
+      store_connection_payload(datum, payload)
+      @instrumenter.start(::Labkit::EXTERNAL_HTTP_NOTIFICATION_TOPIC, payload)
+      @stack.request_call(datum)
+    end
+
+    def response_call(datum)
+      payload = fetch_connection_payload(datum)
+
+      return @stack.response_call(datum) if payload.nil?
+
+      calculate_duration(payload)
+      payload[:code] = datum[:response][:status].to_s
+
+      @instrumenter.finish(::Labkit::EXTERNAL_HTTP_NOTIFICATION_TOPIC, payload)
+      @stack.response_call(datum)
+    ensure
+      remove_connection_payload(datum)
+    end
+
+    def error_call(datum)
+      payload = fetch_connection_payload(datum)
+
+      return @stack.error_call(datum) if payload.nil?
+
+      calculate_duration(payload)
+
+      if datum[:error].is_a?(Exception)
+        payload[:exception] = [datum[:error].class.name, datum[:error].message]
+        payload[:exception_object] = datum[:error]
+      elsif datum[:error].is_a?(String)
+        exception = StandardError.new(datum[:error])
+        payload[:exception] = [exception.class.name, exception.message]
+        payload[:exception_object] = exception
+      end
+
+      @instrumenter.finish(::Labkit::EXTERNAL_HTTP_NOTIFICATION_TOPIC, payload)
+      @stack.error_call(datum)
+    ensure
+      remove_connection_payload(datum)
+    end
+
+    private
+
+    def start_payload(datum)
+      payload = {
+        method: datum[:method].to_s.upcase,
+        host: nil_or_string(datum[:host]),
+        path: nil_or_string(datum[:path]),
+        port: nil_or_int(datum[:port]),
+        scheme: nil_or_string(datum[:scheme]),
+        query: generate_query_string(datum[:query]),
+        start_time: ::Labkit::System.monotonic_time,
+      }
+      unless datum[:proxy].nil?
+        payload[:proxy_host] = datum[:proxy][:host]
+        payload[:proxy_port] = datum[:proxy][:port]
+      end
+      payload
+    end
+
+    def calculate_duration(payload)
+      start_time = payload.delete(:start_time) || ::Labkit::System.monotonic_time
+      payload[:duration] = (::Labkit::System.monotonic_time - start_time).to_f
+    end
+
+    def connection_payload
+      Thread.current[:__labkit_http_excon_payload] ||= {}
+    end
+
+    def store_connection_payload(datum, payload)
+      connection_payload[datum[:connection]] = payload
+    end
+
+    def fetch_connection_payload(datum)
+      connection_payload.fetch(datum[:connection], nil)
+    end
+
+    def remove_connection_payload(datum)
+      connection_payload.delete(datum[:connection])
+    end
+
+    def nil_or_string(str)
+      str&.to_s
+    end
+
+    def nil_or_int(int)
+      int&.to_i
+    rescue
+      nil
+    end
+
+    def generate_query_string(query)
+      if query.is_a?(Hash)
+        query.to_query
+      else
+        nil_or_string(query)
+      end
+    end
+  end
+end

data/lib/labkit/httpclient_publisher.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Labkit
+  ##
+  # Prepend to HTTPClient class to publish an ActiveSupport::Notifcation
+  # whenever a HTTP request is triggered.
+  #
+  # Similar to Net::HTTP, this HTTP client redirects all calls to
+  # HTTPClient#do_get_block. HTTPClient is prepended with HTTPClientPublisher.
+  # Although HTTPClient supports request filter (a kind of middleware), its
+  # support is strictly limited. The request and response passed into the
+  # filter don't contain connection information. The response doesn't even
+  # contain any link to the request object. It's impossible to fit this filter
+  # mechanism into our subscribing model.
+  #
+  # For more information;
+  # https://github.com/nahi/httpclient/blob/d3091b095a1b29f65f4531a70a8e581e75be035e/lib/httpclient.rb#L1233
+  module HTTPClientPublisher
+    @prepend_mutex = Mutex.new
+
+    def self.labkit_prepend!
+      @prepend_mutex.synchronize do
+        return if !defined?(HTTPClient) || @prepended
+
+        HTTPClient.prepend(self)
+        @prepended = true
+      end
+    end
+
+    def do_get_block(req, proxy, conn, &block)
+      start_time = ::Labkit::System.monotonic_time
+      ActiveSupport::Notifications.instrument ::Labkit::EXTERNAL_HTTP_NOTIFICATION_TOPIC, create_request_payload(req, proxy) do |payload|
+        response =
+          begin
+            super
+          ensure
+            payload[:duration] = (::Labkit::System.monotonic_time - start_time).to_f
+          end
+        payload[:code] = response.status_code.to_s
+        response
+      end
+    end
+
+    private
+
+    def create_request_payload(request, proxy)
+      http_header = request.http_header
+      payload = {
+        method: http_header.request_method,
+        host: http_header.request_uri.host,
+        path: http_header.request_uri.path,
+        port: http_header.request_uri.port,
+        scheme: http_header.request_uri.scheme,
+        query: http_header.request_uri.query,
+        fragment: http_header.request_uri.fragment,
+      }
+
+      unless proxy.nil?
+        payload[:proxy_host] = proxy.host
+        payload[:proxy_port] = proxy.port
+      end
+
+      payload
+    end
+  end
+end

data/lib/labkit/logging/grpc/server_interceptor.rb

@@ -55,7 +55,7 @@ module Labkit
 
         private
 
-        def log_request(method, call)
+        def log_request(method, _call)
           start = Time.now
           code = ::GRPC::Core::StatusCodes::OK
 
@@ -77,6 +77,11 @@ module Labkit
             time: Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.%LZ"),
           )
 
+          if ex
+            message["exception"] = ex.message
+            message["exception_backtrace"] = ex.backtrace[0..5] if ex.backtrace
+          end
+
           @log_file.puts(JSON.dump(message))
         end
       end

data/lib/labkit/middleware/sidekiq/context/server.rb

@@ -8,7 +8,8 @@ module Labkit
         # reinstantiate a context in which the job will run.
         class Server
           def call(_worker_class, job, _queue)
-            data = job.merge(Labkit::Context.log_key(:caller_id) => job["class"])
+            worker_name = (job["wrapped"].presence || job["class"]).to_s
+            data = job.merge(Labkit::Context.log_key(:caller_id) => worker_name)
 
             Labkit::Context.with_context(data) do |_context|
               yield

data/lib/labkit/middleware/sidekiq/tracing/client.rb

@@ -15,7 +15,7 @@ module Labkit
           SPAN_KIND = "client"
 
           def call(_worker_class, job, _queue, _redis_pool)
-            Labkit::Tracing::TracingUtils.with_tracing(operation_name: "sidekiq:#{job["class"]}", tags: tags_from_job(job, SPAN_KIND)) do |span|
+            Labkit::Tracing::TracingUtils.with_tracing(operation_name: "sidekiq:#{job_class(job)}", tags: tags_from_job(job, SPAN_KIND)) do |span|
               # Inject the details directly into the job
               Labkit::Tracing::TracingUtils.tracer.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, job)
 

data/lib/labkit/middleware/sidekiq/tracing/server.rb

@@ -17,7 +17,7 @@ module Labkit
           def call(_worker, job, _queue)
             context = Labkit::Tracing::TracingUtils.tracer.extract(OpenTracing::FORMAT_TEXT_MAP, job)
 
-            Labkit::Tracing::TracingUtils.with_tracing(operation_name: "sidekiq:#{job["class"]}", child_of: context, tags: tags_from_job(job, SPAN_KIND)) { |_span| yield }
+            Labkit::Tracing::TracingUtils.with_tracing(operation_name: "sidekiq:#{job_class(job)}", child_of: context, tags: tags_from_job(job, SPAN_KIND)) { |_span| yield }
           end
         end
       end

data/lib/labkit/middleware/sidekiq/tracing/sidekiq_common.rb

@@ -6,15 +6,28 @@ module Labkit
       module Tracing
         # SidekiqCommon is a mixin for the sidekiq middleware components
         module SidekiqCommon
+          def job_class(job)
+            # Active Job wrapping can be found at
+            # https://github.com/rails/rails/blob/v6.0.3.1/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb
+            job["wrapped"].presence || job["class"].presence || "undefined"
+          end
+
+          def wrapped?(job)
+            job["wrapped"].present?
+          end
+
           def tags_from_job(job, kind)
-            {
+            tags = {
               "component" => "sidekiq",
               "span.kind" => kind,
+              "sidekiq.wrapped" => wrapped?(job),
               "sidekiq.queue" => job["queue"],
               "sidekiq.jid" => job["jid"],
               "sidekiq.retry" => job["retry"].to_s,
               "sidekiq.args" => job["args"]&.join(", "),
             }
+            tags["sidekiq.at"] = job["at"] if job["at"]
+            tags
           end
         end
       end

data/lib/labkit/net_http_publisher.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Labkit
+  ##
+  # Prepend to Ruby's Net/HTTP standard HTTP library to publish a notification
+  # whenever a HTTP request is triggered. Net::HTTP has different class methods
+  # for each http method. Those methods are delegated to corresponding instance
+  # methods. Eventually, `request` method is call to dispatch the HTTP request.
+  # Therefore, a prepender that override `request` method covers all HTTP
+  # calls.
+  #
+  # For more information:
+  # https://github.com/ruby/ruby/blob/9b9cbbbc17bb5840581c7da37fd0feb0a7d4c1f3/lib/net/http.rb#L1510
+  #
+  # Note: some use cases to take care of
+  # - Create a request from input URI
+  # - Create a request from input host, port, and path string
+  # - Create a singular request and closes the connection immediately
+  # - Create a persistent connection and perform multiple HTTP requests
+  # - Notification payload must separate URI components
+  # - Create a post request with a body
+  # - Create a post request with form data
+  # - Create a request with basic authentication
+  # - Make a request via a proxy server
+  # - Streaming
+  module NetHttpPublisher
+    @prepend_mutex = Mutex.new
+
+    def self.labkit_prepend!
+      @prepend_mutex.synchronize do
+        return if @prepended
+
+        require "net/http"
+        Net::HTTP.prepend(self)
+        @prepended = true
+      end
+    end
+
+    def request(request, *args, &block)
+      return super unless started?
+
+      start_time = ::Labkit::System.monotonic_time
+
+      ActiveSupport::Notifications.instrument ::Labkit::EXTERNAL_HTTP_NOTIFICATION_TOPIC, create_request_payload(request) do |payload|
+        response =
+          begin
+            super
+          ensure
+            payload[:duration] = (::Labkit::System.monotonic_time - start_time).to_f
+          end
+        payload[:code] = response.code
+        response
+      end
+    end
+
+    private
+
+    def create_request_payload(request)
+      payload = {
+        method: request.method,
+      }
+
+      if request.uri.nil?
+        path_uri = URI(request.path)
+        payload[:host] = address
+        payload[:path] = path_uri.path
+        payload[:port] = port
+        payload[:scheme] = use_ssl? ? "https" : "http"
+        payload[:query] = path_uri.query
+        payload[:fragment] = path_uri.fragment
+      else
+        payload[:host] = request.uri.host
+        payload[:path] = request.uri.path
+        payload[:port] = request.uri.port
+        payload[:scheme] = request.uri.scheme
+        payload[:query] = request.uri.query
+        payload[:fragment] = request.uri.fragment
+      end
+
+      if proxy?
+        payload[:proxy_host] = proxy_address
+        payload[:proxy_port] = proxy_port
+      end
+
+      payload
+    end
+  end
+end

data/lib/labkit/system.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Labkit
+  # A helper class to store system-related methods used in metrics, tracing, and logging
+  module System
+    # Returns the current monotonic clock time as seconds with microseconds precision.
+    #
+    # Returns the time as a Float.
+    def self.monotonic_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second)
+    end
+  end
+end

data/lib/labkit/tracing.rb

@@ -5,6 +5,8 @@ require "active_support/all"
 module Labkit
   # Tracing provides distributed tracing functionality
   module Tracing
+    autoload :AbstractInstrumenter, "labkit/tracing/abstract_instrumenter"
+    autoload :TracingCommon, "labkit/tracing/tracing_common"
     autoload :Factory, "labkit/tracing/factory"
     autoload :GRPC, "labkit/tracing/grpc"
     autoload :GRPCInterceptor, "labkit/tracing/grpc_interceptor" # Deprecated
@@ -12,6 +14,7 @@ module Labkit
     autoload :RackMiddleware, "labkit/tracing/rack_middleware"
     autoload :Rails, "labkit/tracing/rails"
     autoload :Redis, "labkit/tracing/redis"
+    autoload :ExternalHttp, "labkit/tracing/external_http"
     autoload :Sidekiq, "labkit/tracing/sidekiq"
     autoload :TracingUtils, "labkit/tracing/tracing_utils"
 
@@ -31,7 +34,7 @@ module Labkit
     # Check if the current request is being traced.
     def self.sampled?
       context = OpenTracing.active_span&.context
-      context && context.respond_to?(:sampled?) && context.sampled?
+      context&.respond_to?(:sampled?) && context&.sampled?
     end
 
     def self.stacktrace_operations

data/lib/labkit/tracing/abstract_instrumenter.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "opentracing"
+require "active_support/all"
+
+module Labkit
+  module Tracing
+    # https://edgeapi.rubyonrails.org/classes/ActiveSupport/Notifications/Instrumenter.html#method-c-new
+    class AbstractInstrumenter
+      def start(_name, _id, payload)
+        scope = OpenTracing.start_active_span(span_name(payload))
+
+        scope_stack.push scope
+      end
+
+      def finish(_name, _id, payload)
+        scope = scope_stack.pop
+        span = scope.span
+
+        Labkit::Tracing::TracingUtils.log_common_fields_on_span(span, span_name(payload))
+
+        # exception_object is the standard exception payload from ActiveSupport::Notifications
+        # https://github.com/rails/rails/blob/v6.0.3.1/activesupport/lib/active_support/notifications/instrumenter.rb#L26
+        exception = payload[:exception_object].presence || payload[:exception].presence
+        Labkit::Tracing::TracingUtils.log_exception_on_span(span, exception)
+
+        tags(payload).each do |k, v|
+          span.set_tag(k, v)
+        end
+
+        scope.close
+      end
+
+      def scope_stack
+        Thread.current[:_labkit_trace_scope_stack] ||= []
+      end
+
+      def span_name(_payload)
+        raise "span_name not implemented"
+      end
+
+      def tags(_payload)
+        {}
+      end
+    end
+  end
+end

data/lib/labkit/tracing/external_http.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    # Instrument external HTTP calls made by the HTTP client libraries. This
+    # tracing instrumenter listens to the events broadcasted from the
+    # publishers injected into the libraries whenever there is a request.
+    module ExternalHttp
+      include Labkit::Tracing::TracingCommon
+
+      autoload :RequestInstrumenter, "labkit/tracing/external_http/request_instrumenter"
+
+      def self.instrument
+        Labkit::NetHttpPublisher.labkit_prepend!
+        Labkit::ExconPublisher.labkit_prepend!
+        Labkit::HTTPClientPublisher.labkit_prepend!
+
+        subscriptions = [
+          ::ActiveSupport::Notifications.subscribe(::Labkit::EXTERNAL_HTTP_NOTIFICATION_TOPIC, RequestInstrumenter.new),
+        ]
+
+        create_unsubscriber subscriptions
+      end
+    end
+  end
+end

data/lib/labkit/tracing/external_http/request_instrumenter.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Tracing
+    module ExternalHttp
+      # For more information on the payloads: lib/labkit/net_http_publisher.rb
+      class RequestInstrumenter < Labkit::Tracing::AbstractInstrumenter
+        def span_name(_payload)
+          "external_http:request"
+        end
+
+        def tags(payload)
+          # Duration is calculated by start and end time
+          # Exception is already captured in lib/labkit/tracing/tracing_utils.rb
+          tags = {
+            "component" => "external_http",
+            "method" => payload[:method],
+            "code" => payload[:code],
+            "host" => payload[:host],
+            "port" => payload[:port],
+            "path" => payload[:path],
+            "scheme" => payload[:scheme],
+          }
+          unless payload[:proxy_host].nil?
+            tags["proxy_host"] = payload[:proxy_host]
+            tags["proxy_port"] = payload[:proxy_port]
+          end
+          tags
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/rails.rb

@@ -4,11 +4,9 @@ module Labkit
   module Tracing
     # Rails provides classes for instrumenting Rails events
     module Rails
-      autoload :AbstractInstrumenter, "labkit/tracing/rails/abstract_instrumenter"
       autoload :ActionView, "labkit/tracing/rails/action_view"
       autoload :ActiveRecord, "labkit/tracing/rails/active_record"
       autoload :ActiveSupport, "labkit/tracing/rails/active_support"
-      autoload :RailsCommon, "labkit/tracing/rails/rails_common"
 
       ActionViewSubscriber = ActionView::Subscriber
       ActiveRecordSubscriber = ActiveRecord::Subscriber

data/lib/labkit/tracing/rails/action_view.rb

@@ -10,6 +10,20 @@ module Labkit
         autoload :Subscriber, "labkit/tracing/rails/action_view/subscriber"
 
         COMPONENT_TAG = "ActionView"
+
+        # Returns identifier relative to Rails.root. Rails supports different template types and returns corresponding identifiers:
+        # - Text template: the identifier is "text template"
+        # - Html template: the identifier is "html template"
+        # - Inline template: the identifier is "inline template"
+        # - Raw template: the identifier is the file path of the template
+        # Therefore, the amount of returned identifiers is static.
+        def self.template_identifier(payload)
+          return if !defined?(::Rails.root) || payload[:identifier].nil?
+
+          # Rails.root returns a Pathname object, whose `to_s` methods returns an absolute path without ending "/"
+          # Source: https://github.com/rails/rails/blob/v6.0.3.1/railties/lib/rails.rb#L64
+          payload[:identifier].sub("#{::Rails.root}/", "")
+        end
       end
     end
   end

data/lib/labkit/tracing/rails/action_view/render_collection_instrumenter.rb

@@ -5,9 +5,14 @@ module Labkit
     module Rails
       module ActionView
         # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
-        class RenderCollectionInstrumenter < AbstractInstrumenter
+        class RenderCollectionInstrumenter < Labkit::Tracing::AbstractInstrumenter
           def span_name(payload)
-            "render_collection"
+            identifier = ActionView.template_identifier(payload)
+            if identifier.nil?
+              "render_collection"
+            else
+              "render_collection:#{identifier}"
+            end
           end
 
           def tags(payload)

data/lib/labkit/tracing/rails/action_view/render_partial_instrumenter.rb

@@ -5,9 +5,14 @@ module Labkit
     module Rails
       module ActionView
         # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
-        class RenderPartialInstrumenter < AbstractInstrumenter
+        class RenderPartialInstrumenter < Labkit::Tracing::AbstractInstrumenter
           def span_name(payload)
-            "render_partial"
+            identifier = ActionView.template_identifier(payload)
+            if identifier.nil?
+              "render_partial"
+            else
+              "render_partial:#{identifier}"
+            end
           end
 
           def tags(payload)

data/lib/labkit/tracing/rails/action_view/render_template_instrumenter.rb

@@ -5,9 +5,14 @@ module Labkit
     module Rails
       module ActionView
         # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
-        class RenderTemplateInstrumenter < AbstractInstrumenter
+        class RenderTemplateInstrumenter < Labkit::Tracing::AbstractInstrumenter
           def span_name(payload)
-            "render_template"
+            identifier = ActionView.template_identifier(payload)
+            if identifier.nil?
+              "render_template"
+            else
+              "render_template:#{identifier}"
+            end
           end
 
           def tags(payload)

data/lib/labkit/tracing/rails/action_view/subscriber.rb

@@ -7,7 +7,7 @@ module Labkit
         # ActionView bridges action view notifications to
         # the distributed tracing subsystem
         class Subscriber
-          include RailsCommon
+          include Labkit::Tracing::TracingCommon
 
           RENDER_TEMPLATE_NOTIFICATION_TOPIC = "render_template.action_view"
           RENDER_COLLECTION_NOTIFICATION_TOPIC = "render_collection.action_view"

data/lib/labkit/tracing/rails/active_record/sql_instrumenter.rb

@@ -5,7 +5,7 @@ module Labkit
     module Rails
       module ActiveRecord
         # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
-        class SqlInstrumenter < AbstractInstrumenter
+        class SqlInstrumenter < Labkit::Tracing::AbstractInstrumenter
           OPERATION_NAME_PREFIX = "active_record:"
           DEFAULT_OPERATION_NAME = "sqlquery"
 

data/lib/labkit/tracing/rails/active_record/subscriber.rb

@@ -7,7 +7,7 @@ module Labkit
         # ActiveRecord bridges active record notifications to
         # the distributed tracing subsystem
         class Subscriber
-          include RailsCommon
+          include Labkit::Tracing::TracingCommon
 
           ACTIVE_RECORD_NOTIFICATION_TOPIC = "sql.active_record"
 

data/lib/labkit/tracing/rails/active_support/cache_delete_instrumenter.rb

@@ -5,7 +5,7 @@ module Labkit
     module Rails
       module ActiveSupport
         # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
-        class CacheDeleteInstrumenter < AbstractInstrumenter
+        class CacheDeleteInstrumenter < Labkit::Tracing::AbstractInstrumenter
           def span_name(payload)
             "cache_delete"
           end

data/lib/labkit/tracing/rails/active_support/cache_fetch_hit_instrumenter.rb

@@ -5,7 +5,7 @@ module Labkit
     module Rails
       module ActiveSupport
         # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
-        class CacheFetchHitInstrumenter < AbstractInstrumenter
+        class CacheFetchHitInstrumenter < Labkit::Tracing::AbstractInstrumenter
           def span_name(payload)
             "cache_fetch_hit"
           end

data/lib/labkit/tracing/rails/active_support/cache_generate_instrumenter.rb

@@ -5,7 +5,7 @@ module Labkit
     module Rails
       module ActiveSupport
         # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
-        class CacheGenerateInstrumenter < AbstractInstrumenter
+        class CacheGenerateInstrumenter < Labkit::Tracing::AbstractInstrumenter
           def span_name(payload)
             "cache_generate"
           end

data/lib/labkit/tracing/rails/active_support/cache_read_instrumenter.rb

@@ -5,7 +5,7 @@ module Labkit
     module Rails
       module ActiveSupport
         # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
-        class CacheReadInstrumenter < AbstractInstrumenter
+        class CacheReadInstrumenter < Labkit::Tracing::AbstractInstrumenter
           def span_name(payload)
             "cache_read"
           end

data/lib/labkit/tracing/rails/active_support/cache_write_instrumenter.rb

@@ -5,7 +5,7 @@ module Labkit
     module Rails
       module ActiveSupport
         # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
-        class CacheWriteInstrumenter < AbstractInstrumenter
+        class CacheWriteInstrumenter < Labkit::Tracing::AbstractInstrumenter
           def span_name(payload)
             "cache_write"
           end

data/lib/labkit/tracing/rails/active_support/subscriber.rb

@@ -7,7 +7,7 @@ module Labkit
         # ActiveSupport bridges action active support notifications to
         # the distributed tracing subsystem
         class Subscriber
-          include RailsCommon
+          include Labkit::Tracing::TracingCommon
 
           CACHE_READ_TOPIC = "cache_read.active_support"
           CACHE_GENERATE_TOPIC = "cache_generate.active_support"

data/lib/labkit/tracing/tracing_common.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "active_support/all"
+
+module Labkit
+  module Tracing
+    # TracingCommon is a mixin for providing instrumentation
+    # functionality for the instrumentation classes based on
+    # ActiveSupport::Notifications
+    module TracingCommon
+      extend ::ActiveSupport::Concern
+
+      class_methods do
+        def create_unsubscriber(subscriptions)
+          -> { subscriptions.each { |subscriber| ::ActiveSupport::Notifications.unsubscribe(subscriber) } }
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing/tracing_utils.rb

@@ -48,6 +48,8 @@ module Labkit
 
       # Add exception logging to a span
       def self.log_exception_on_span(span, exception)
+        return if exception.blank?
+
         span.set_tag("error", true)
         span.log_kv(**kv_tags_for_exception(exception))
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-labkit
 version: !ruby/object:Gem::Version
-  version: 0.13.5
+  version: 0.16.2
 platform: ruby
 authors:
 - Andrew Newdigate
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-21 00:00:00.000000000 Z
+date: 2021-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -19,7 +19,7 @@ dependencies:
         version: 5.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 6.1.0
+        version: 7.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: 5.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 6.1.0
+        version: 7.0.0
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -39,7 +39,7 @@ dependencies:
         version: 5.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 6.1.0
+        version: 7.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -49,7 +49,7 @@ dependencies:
         version: 5.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 6.1.0
+        version: 7.0.0
 - !ruby/object:Gem::Dependency
   name: grpc
   requirement: !ruby/object:Gem::Requirement
@@ -92,6 +92,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: pg_query
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -113,19 +127,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 5.0.0
 - !ruby/object:Gem::Dependency
-  name: gitlab-pg_query
+  name: excon
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
-  type: :runtime
+        version: 0.78.1
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 0.78.1
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: grpc-tools
   requirement: !ruby/object:Gem::Requirement
@@ -140,6 +168,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.19'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.17.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.17.3
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.3
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -182,6 +238,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -275,6 +345,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".gitlab-ci.yml"
+- ".gitlab/CODEOWNERS"
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-version"
@@ -292,6 +363,8 @@ files:
 - lib/labkit/correlation/grpc/client_interceptor.rb
 - lib/labkit/correlation/grpc/grpc_common.rb
 - lib/labkit/correlation/grpc/server_interceptor.rb
+- lib/labkit/excon_publisher.rb
+- lib/labkit/httpclient_publisher.rb
 - lib/labkit/logging.rb
 - lib/labkit/logging/grpc.rb
 - lib/labkit/logging/grpc/server_interceptor.rb
@@ -308,7 +381,12 @@ files:
 - lib/labkit/middleware/sidekiq/tracing/client.rb
 - lib/labkit/middleware/sidekiq/tracing/server.rb
 - lib/labkit/middleware/sidekiq/tracing/sidekiq_common.rb
+- lib/labkit/net_http_publisher.rb
+- lib/labkit/system.rb
 - lib/labkit/tracing.rb
+- lib/labkit/tracing/abstract_instrumenter.rb
+- lib/labkit/tracing/external_http.rb
+- lib/labkit/tracing/external_http/request_instrumenter.rb
 - lib/labkit/tracing/factory.rb
 - lib/labkit/tracing/grpc.rb
 - lib/labkit/tracing/grpc/client_interceptor.rb
@@ -317,7 +395,6 @@ files:
 - lib/labkit/tracing/jaeger_factory.rb
 - lib/labkit/tracing/rack_middleware.rb
 - lib/labkit/tracing/rails.rb
-- lib/labkit/tracing/rails/abstract_instrumenter.rb
 - lib/labkit/tracing/rails/action_view.rb
 - lib/labkit/tracing/rails/action_view/render_collection_instrumenter.rb
 - lib/labkit/tracing/rails/action_view/render_partial_instrumenter.rb
@@ -333,10 +410,10 @@ files:
 - lib/labkit/tracing/rails/active_support/cache_read_instrumenter.rb
 - lib/labkit/tracing/rails/active_support/cache_write_instrumenter.rb
 - lib/labkit/tracing/rails/active_support/subscriber.rb
-- lib/labkit/tracing/rails/rails_common.rb
 - lib/labkit/tracing/redis.rb
 - lib/labkit/tracing/redis/redis_interceptor.rb
 - lib/labkit/tracing/redis/redis_interceptor_helper.rb
+- lib/labkit/tracing/tracing_common.rb
 - lib/labkit/tracing/tracing_utils.rb
 homepage: https://gitlab.com/gitlab-org/labkit-ruby
 licenses:

data/lib/labkit/tracing/rails/abstract_instrumenter.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-require "active_support/all"
-
-module Labkit
-  module Tracing
-    module Rails
-      # https://edgeapi.rubyonrails.org/classes/ActiveSupport/Notifications/Instrumenter.html#method-c-new
-      class AbstractInstrumenter
-        def start(name, id, payload)
-          scope = OpenTracing.start_active_span(span_name(payload))
-
-          scope_stack.push scope
-        end
-
-        def finish(name, id, payload)
-          scope = scope_stack.pop
-          span = scope.span
-
-          Labkit::Tracing::TracingUtils.log_common_fields_on_span(span, span_name(payload))
-
-          exception = payload[:exception]
-          Labkit::Tracing::TracingUtils.log_exception_on_span(span, exception) if exception
-
-          tags(payload).each do |k, v|
-            span.set_tag(k, v)
-          end
-
-          scope.close
-        end
-
-        def scope_stack
-          Thread.current[:_labkit_trace_scope_stack] ||= []
-        end
-
-        def span_name(payload)
-          raise "span_name not implemented"
-        end
-
-        def tags(payload)
-          {}
-        end
-      end
-    end
-  end
-end

data/lib/labkit/tracing/rails/rails_common.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-require "active_support/all"
-
-module Labkit
-  module Tracing
-    module Rails
-      # RailsCommon is a mixin for providing instrumentation
-      # functionality for the rails instrumentation classes
-      module RailsCommon
-        extend ::ActiveSupport::Concern
-
-        class_methods do
-          def create_unsubscriber(subscriptions)
-            -> { subscriptions.each { |subscriber| ::ActiveSupport::Notifications.unsubscribe(subscriber) } }
-          end
-        end
-      end
-    end
-  end
-end