gitlab-labkit 0.10.1 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2476531528be243ed0069feffe3bbfbc7d557556455616e7d4216aa686422409
-  data.tar.gz: 43223820267f18725601573595ba607178788178b1922835becaf7c72d593b2f
+  metadata.gz: 54ad7f3ed011ed85b6f0c090f05cd47a4bf7c738fabd2a165bad85b6e4d288c0
+  data.tar.gz: c833a07fa745fe8286b9c000879de8b0f338c0d24b722e71dc5044c60dcc3b79
 SHA512:
-  metadata.gz: 712fd95e6a754409e83a8b5ee690c1843b233794a33a7461dea2fdc84c49eb8e873c7298ee15087302c32b60faeca83fd3592a80e471640a3c80710fa094e54e
-  data.tar.gz: e6d4d048caa06702bd47326574d105f32d9d5e7c27d3c619ac529e4cb35f671a7cfca92e1d9b6460a1b6080c90bc05306f2df17381ca6e104ad30a5171a22bb8
+  metadata.gz: 5c724e7322119314f35831c726900220842e2f473a48d81241a94cc2eb4f34da39bb38c843ba72b724a1b5d0dff5e1b115cec704468d49797d4c681c309cf7ba
+  data.tar.gz: 91e870d9af690c2b5f6ce7659954616880ee3313f710b49c89505cbe915c4606ef3d1032d905cbfd1be4e547d98c846baa6fed052836833e71dc7a59444c0727

data/lib/labkit/context.rb

@@ -21,7 +21,8 @@ module Labkit
     LOG_KEY = "meta"
     CORRELATION_ID_KEY = "correlation_id"
     RAW_KEYS = [CORRELATION_ID_KEY].freeze
-    KNOWN_KEYS = %w[user project root_namespace subscription_plan caller_id].freeze
+    KNOWN_KEYS = %w[user project root_namespace subscription_plan caller_id
+                    related_class feature_category].freeze
 
     class << self
       def with_context(attributes = {})

data/lib/labkit/correlation/grpc/grpc_common.rb

@@ -7,6 +7,16 @@ module Labkit
       # It is not part of the public API
       module GRPCCommon
         CORRELATION_METADATA_KEY = "x-gitlab-correlation-id"
+
+        def rpc_split(method)
+          owner = method.owner
+          method_name, = owner.rpc_descs.find do |k, _|
+            ::GRPC::GenericService.underscore(k.to_s) == method.name.to_s
+          end
+          method_name ||= "(unknown)"
+
+          [owner.service_name, method_name]
+        end
       end
     end
   end

data/lib/labkit/logging.rb

@@ -4,6 +4,7 @@ module Labkit
   # Logging provides functionality for logging, such as
   # sanitization
   module Logging
+    autoload :GRPC, "labkit/logging/grpc"
     autoload :Sanitizer, "labkit/logging/sanitizer"
   end
 end

data/lib/labkit/logging/grpc.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Logging
+    module GRPC
+      autoload :ServerInterceptor, "labkit/logging/grpc/server_interceptor"
+    end
+  end
+end

data/lib/labkit/logging/grpc/server_interceptor.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "grpc"
+require "json"
+
+module Labkit
+  module Logging
+    module GRPC
+      class ServerInterceptor < ::GRPC::ServerInterceptor
+        include Labkit::Correlation::GRPC::GRPCCommon
+
+        CODE_STRINGS = {
+          ::GRPC::Core::StatusCodes::OK => "OK",
+          ::GRPC::Core::StatusCodes::CANCELLED => "Canceled",
+          ::GRPC::Core::StatusCodes::UNKNOWN => "Unknown",
+          ::GRPC::Core::StatusCodes::INVALID_ARGUMENT => "InvalidArgument",
+          ::GRPC::Core::StatusCodes::DEADLINE_EXCEEDED => "DeadlineExceeded",
+          ::GRPC::Core::StatusCodes::NOT_FOUND => "NotFound",
+          ::GRPC::Core::StatusCodes::ALREADY_EXISTS => "AlreadyExists",
+          ::GRPC::Core::StatusCodes::PERMISSION_DENIED => "PermissionDenied",
+          ::GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED => "ResourceExhausted",
+          ::GRPC::Core::StatusCodes::FAILED_PRECONDITION => "FailedPrecondition",
+          ::GRPC::Core::StatusCodes::ABORTED => "Aborted",
+          ::GRPC::Core::StatusCodes::OUT_OF_RANGE => "OutOfRange",
+          ::GRPC::Core::StatusCodes::UNIMPLEMENTED => "Unimplemented",
+          ::GRPC::Core::StatusCodes::INTERNAL => "Internal",
+          ::GRPC::Core::StatusCodes::UNAVAILABLE => "Unavailable",
+          ::GRPC::Core::StatusCodes::DATA_LOSS => "DataLoss",
+          ::GRPC::Core::StatusCodes::UNAUTHENTICATED => "Unauthenticated",
+        }.freeze
+
+        def initialize(log_file, default_tags)
+          @log_file = log_file
+          @log_file.sync = true
+          @default_tags = default_tags
+
+          super()
+        end
+
+        def request_response(request: nil, call: nil, method: nil)
+          log_request(method, call) { yield }
+        end
+
+        def server_streamer(request: nil, call: nil, method: nil)
+          log_request(method, call) { yield }
+        end
+
+        def client_streamer(call: nil, method: nil)
+          log_request(method, call) { yield }
+        end
+
+        def bidi_streamer(requests: nil, call: nil, method: nil)
+          log_request(method, call) { yield }
+        end
+
+        private
+
+        def log_request(method, call)
+          start = Time.now
+          code = ::GRPC::Core::StatusCodes::OK
+
+          yield
+        rescue StandardError => ex
+          code = ex.is_a?(::GRPC::BadStatus) ? ex.code : ::GRPC::Core::StatusCodes::UNKNOWN
+
+          raise
+        ensure
+          service_name, method_name = rpc_split(method)
+          message = @default_tags.merge(
+            'grpc.time_ms': ((Time.now - start) * 1000.0).truncate(3),
+            'grpc.code': CODE_STRINGS.fetch(code, code.to_s),
+            'grpc.method': method_name,
+            'grpc.service': service_name,
+            pid: Process.pid,
+            correlation_id: Labkit::Correlation::CorrelationId.current_id.to_s,
+            time: Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.%LZ"),
+          )
+
+          @log_file.puts(JSON.dump(message))
+        end
+      end
+    end
+  end
+end

data/lib/labkit/tracing.rb

@@ -28,6 +28,10 @@ module Labkit
       ENV["GITLAB_TRACING_URL"]
     end
 
+    def self.stacktrace_operations
+      @stacktrace_operations ||= Set.new(ENV["GITLAB_TRACING_INCLUDE_STACKTRACE"].to_s.split(",").map(&:strip))
+    end
+
     def self.tracing_url_enabled?
       enabled? && tracing_url_template.present?
     end

data/lib/labkit/tracing/grpc/server_interceptor.rb

@@ -14,6 +14,8 @@ module Labkit
       # for instrumenting GRPC calls with distributed tracing
       # in a GRPC Ruby server
       class ServerInterceptor < ::GRPC::ServerInterceptor
+        include Labkit::Correlation::GRPC::GRPCCommon
+
         def request_response(request: nil, call: nil, method: nil)
           wrap_with_tracing(call, method, "unary") do
             yield
@@ -40,16 +42,9 @@ module Labkit
 
         private
 
-        def route_from_method(method)
-          service_class = method.owner
-          rpc_method = method.name.to_s.split("_").map(&:capitalize).join("")
-
-          "/#{service_class.service_name}/#{rpc_method}"
-        end
-
         def wrap_with_tracing(call, method, grpc_type)
           context = TracingUtils.tracer.extract(OpenTracing::FORMAT_TEXT_MAP, call.metadata)
-          method_name = route_from_method(method)
+          method_name = "/#{rpc_split(method).join("/")}"
           tags = {
             "component" => "grpc",
             "span.kind" => "server",

data/lib/labkit/tracing/rails/abstract_instrumenter.rb

@@ -17,6 +17,8 @@ module Labkit
           scope = scope_stack.pop
           span = scope.span
 
+          Labkit::Tracing::TracingUtils.log_common_fields_on_span(span, span_name(payload))
+
           exception = payload[:exception]
           Labkit::Tracing::TracingUtils.log_exception_on_span(span, exception) if exception
 

data/lib/labkit/tracing/redis/redis_interceptor_helper.rb

@@ -8,7 +8,7 @@ module Labkit
       # RedisInterceptorHelper is a helper for the RedisInterceptor. This is not a public API
       class RedisInterceptorHelper
         # For optimization, compile this once
-        MASK_REDIS_RE = /^([\w-]+(?:\W+[\w-]+(?:\W+[\w-]+)?)?)(.?)/.freeze
+        MASK_REDIS_RE = /^([\w{}-]+(?:\W+[\w{}-]+(?:\W+[\w{}-]+)?)?)(.?)/.freeze
 
         def self.call_with_tracing(command, client)
           Labkit::Tracing::TracingUtils.with_tracing(operation_name: "redis.call", tags: tags_from_command(command, client)) do |_span|
@@ -99,6 +99,7 @@ module Labkit
           return "" if argument.empty?
 
           matches = argument.match(MASK_REDIS_RE)
+
           matches[2].empty? ? matches[0] : matches[0] + "*****"
         end
         private_class_method :mask_redis_arg

data/lib/labkit/tracing/tracing_utils.rb

@@ -12,9 +12,7 @@ module Labkit
         scope = tracer.start_active_span(operation_name, child_of: child_of, tags: tags)
         span = scope.span
 
-        # Add correlation details to the span if we have them
-        correlation_id = Labkit::Correlation::CorrelationId.current_id
-        span.set_tag("correlation_id", correlation_id) if correlation_id
+        log_common_fields_on_span(span, operation_name)
 
         begin
           yield span
@@ -35,11 +33,19 @@ module Labkit
       def self.postnotify_span(operation_name, start_time, end_time, tags: nil, child_of: nil, exception: nil)
         span = OpenTracing.start_span(operation_name, start_time: start_time, tags: tags, child_of: child_of)
 
+        log_common_fields_on_span(span, operation_name)
         log_exception_on_span(span, exception) if exception
 
         span.finish(end_time: end_time)
       end
 
+      # Add common fields to a span
+      def self.log_common_fields_on_span(span, operation_name)
+        correlation_id = Labkit::Correlation::CorrelationId.current_id
+        span.set_tag("correlation_id", correlation_id) if correlation_id
+        span.log_kv(stack: caller.join('\n')) if include_stacktrace?(operation_name)
+      end
+
       # Add exception logging to a span
       def self.log_exception_on_span(span, exception)
         span.set_tag("error", true)
@@ -60,6 +66,14 @@ module Labkit
           { :"event" => "error", :"error.kind" => exception.class.to_s, :"error.object" => Labkit::Logging::Sanitizer.sanitize_field(exception.to_s) }
         end
       end
+
+      def self.include_stacktrace?(operation_name)
+        @include_stacktrace ||= Hash.new do |result, name|
+          result[name] = Tracing.stacktrace_operations.any? { |stacktrace_operation| name.starts_with?(stacktrace_operation) }
+        end
+
+        @include_stacktrace[operation_name]
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-labkit
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.13.0
 platform: ruby
 authors:
 - Andrew Newdigate
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-05 00:00:00.000000000 Z
+date: 2020-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -279,6 +279,8 @@ files:
 - lib/labkit/correlation/grpc/grpc_common.rb
 - lib/labkit/correlation/grpc/server_interceptor.rb
 - lib/labkit/logging.rb
+- lib/labkit/logging/grpc.rb
+- lib/labkit/logging/grpc/server_interceptor.rb
 - lib/labkit/logging/sanitizer.rb
 - lib/labkit/middleware.rb
 - lib/labkit/middleware/rack.rb