gitlab-labkit 0.12.1 → 0.13.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3267849e09cf37b1f72c5c7cf0f25ac43066edd06b09ff7b8a3b93f2ee5a4f45
-  data.tar.gz: ae8f6710ada3fc3791ae54cba82a4f8b73c5d67003782285264246032c22328e
+  metadata.gz: f7365b2f12b026f6b497d56738197b66e34014676dc284f568e97fad669cecef
+  data.tar.gz: ca9da033595aed9506edd7ba376a1bdd0b15c8e880a5e6b3edc52dc72b5a5570
 SHA512:
-  metadata.gz: 33ec95dfe61241476125e718e9ae3d3145ba2d76244076cfbe164dd9b61b841d1e832dc27ef71846476ca32abe1057165f35c2b44abb3ff83bd09693a354f00b
-  data.tar.gz: cb7abc95873cae4e981348c28533f3c894e50b696ed87822cca9b65a6db689a4e0453b83297b667eaa88c7a879e80dfc520e56e75959431a1388bc1db622aaf3
+  metadata.gz: 87aaa07295098883563e77694fdb1401ba9e2e5fbc26569ee0b87f8ce7bc329f0d26b78df5d2e1c2e97b86f3d7ec3c41200406eb2b7028aba8ce8bdec4001ab4
+  data.tar.gz: 42f5dcee07855cd347b9585b96e44ac3e36e05a5fd634db55ddea02676b82df9c35e0ba3d6bbbbf0e4c6b4ec16906ff8a86f041db67d392485008d811f3d20b2

data/.gitlab-ci.yml

@@ -1,3 +1,12 @@
+workflow:
+  rules:
+    # For merge requests, create a pipeline.
+    - if: '$CI_MERGE_REQUEST_IID'
+    # For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    # For tags, create a pipeline.
+    - if: '$CI_COMMIT_TAG'
+
 .test_template: &test_definition
   stage: test
   script:
@@ -5,6 +14,10 @@
     - bundle install
     - bundle exec rake verify build install
 
+test:2.7:
+  image: ruby:2.7
+  <<: *test_definition
+
 test:2.6:
   image: ruby:2.6
   <<: *test_definition
@@ -13,10 +26,6 @@ test:2.5:
   image: ruby:2.5
   <<: *test_definition
 
-test:2.4:
-  image: ruby:2.4
-  <<: *test_definition
-
 deploy:
   stage: deploy
   script:

data/gitlab-labkit.gemspec

@@ -22,9 +22,10 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "actionpack", ">= 5.0.0", "< 6.1.0"
   spec.add_runtime_dependency "activesupport", ">= 5.0.0", "< 6.1.0"
   spec.add_runtime_dependency "grpc", "~> 1.19" # Be sure to update the "grpc-tools" dev_depenency too
-  spec.add_runtime_dependency "jaeger-client", "~> 0.10"
+  spec.add_runtime_dependency "jaeger-client", "~> 1.1"
   spec.add_runtime_dependency "opentracing", "~> 0.4"
   spec.add_runtime_dependency "redis", ">3.0.0", "<5.0.0"
+  spec.add_runtime_dependency "gitlab-pg_query", "~> 1.3"
 
   # Please maintain alphabetical order for dev dependencies
   spec.add_development_dependency "grpc-tools", "~> 1.19"

data/lib/labkit/context.rb

@@ -22,7 +22,7 @@ module Labkit
     CORRELATION_ID_KEY = "correlation_id"
     RAW_KEYS = [CORRELATION_ID_KEY].freeze
     KNOWN_KEYS = %w[user project root_namespace subscription_plan caller_id
-                    related_class].freeze
+                    related_class feature_category].freeze
 
     class << self
       def with_context(attributes = {})

data/lib/labkit/correlation/grpc/grpc_common.rb

@@ -7,6 +7,16 @@ module Labkit
       # It is not part of the public API
       module GRPCCommon
         CORRELATION_METADATA_KEY = "x-gitlab-correlation-id"
+
+        def rpc_split(method)
+          owner = method.owner
+          method_name, = owner.rpc_descs.find do |k, _|
+            ::GRPC::GenericService.underscore(k.to_s) == method.name.to_s
+          end
+          method_name ||= "(unknown)"
+
+          [owner.service_name, method_name]
+        end
       end
     end
   end

data/lib/labkit/logging.rb

@@ -4,6 +4,7 @@ module Labkit
   # Logging provides functionality for logging, such as
   # sanitization
   module Logging
+    autoload :GRPC, "labkit/logging/grpc"
     autoload :Sanitizer, "labkit/logging/sanitizer"
   end
 end

data/lib/labkit/logging/grpc.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Labkit
+  module Logging
+    module GRPC
+      autoload :ServerInterceptor, "labkit/logging/grpc/server_interceptor"
+    end
+  end
+end

data/lib/labkit/logging/grpc/server_interceptor.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "grpc"
+require "json"
+
+module Labkit
+  module Logging
+    module GRPC
+      class ServerInterceptor < ::GRPC::ServerInterceptor
+        include Labkit::Correlation::GRPC::GRPCCommon
+
+        CODE_STRINGS = {
+          ::GRPC::Core::StatusCodes::OK => "OK",
+          ::GRPC::Core::StatusCodes::CANCELLED => "Canceled",
+          ::GRPC::Core::StatusCodes::UNKNOWN => "Unknown",
+          ::GRPC::Core::StatusCodes::INVALID_ARGUMENT => "InvalidArgument",
+          ::GRPC::Core::StatusCodes::DEADLINE_EXCEEDED => "DeadlineExceeded",
+          ::GRPC::Core::StatusCodes::NOT_FOUND => "NotFound",
+          ::GRPC::Core::StatusCodes::ALREADY_EXISTS => "AlreadyExists",
+          ::GRPC::Core::StatusCodes::PERMISSION_DENIED => "PermissionDenied",
+          ::GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED => "ResourceExhausted",
+          ::GRPC::Core::StatusCodes::FAILED_PRECONDITION => "FailedPrecondition",
+          ::GRPC::Core::StatusCodes::ABORTED => "Aborted",
+          ::GRPC::Core::StatusCodes::OUT_OF_RANGE => "OutOfRange",
+          ::GRPC::Core::StatusCodes::UNIMPLEMENTED => "Unimplemented",
+          ::GRPC::Core::StatusCodes::INTERNAL => "Internal",
+          ::GRPC::Core::StatusCodes::UNAVAILABLE => "Unavailable",
+          ::GRPC::Core::StatusCodes::DATA_LOSS => "DataLoss",
+          ::GRPC::Core::StatusCodes::UNAUTHENTICATED => "Unauthenticated",
+        }.freeze
+
+        def initialize(log_file, default_tags)
+          @log_file = log_file
+          @log_file.sync = true
+          @default_tags = default_tags
+
+          super()
+        end
+
+        def request_response(request: nil, call: nil, method: nil)
+          log_request(method, call) { yield }
+        end
+
+        def server_streamer(request: nil, call: nil, method: nil)
+          log_request(method, call) { yield }
+        end
+
+        def client_streamer(call: nil, method: nil)
+          log_request(method, call) { yield }
+        end
+
+        def bidi_streamer(requests: nil, call: nil, method: nil)
+          log_request(method, call) { yield }
+        end
+
+        private
+
+        def log_request(method, call)
+          start = Time.now
+          code = ::GRPC::Core::StatusCodes::OK
+
+          yield
+        rescue StandardError => ex
+          code = ex.is_a?(::GRPC::BadStatus) ? ex.code : ::GRPC::Core::StatusCodes::UNKNOWN
+
+          raise
+        ensure
+          service_name, method_name = rpc_split(method)
+          message = @default_tags.merge(
+            'grpc.start_time': start.utc.rfc3339,
+            'grpc.time_ms': ((Time.now - start) * 1000.0).truncate(3),
+            'grpc.code': CODE_STRINGS.fetch(code, code.to_s),
+            'grpc.method': method_name,
+            'grpc.service': service_name,
+            pid: Process.pid,
+            correlation_id: Labkit::Correlation::CorrelationId.current_id.to_s,
+            time: Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.%LZ"),
+          )
+
+          @log_file.puts(JSON.dump(message))
+        end
+      end
+    end
+  end
+end

data/lib/labkit/logging/sanitizer.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "pg_query"
+
 module Labkit
   module Logging
     # Sanitizer provides log message sanitization, removing
@@ -22,6 +24,12 @@ module Labkit
         content
       end
 
+      def self.sanitize_sql(sql)
+        PgQuery.normalize(sql)
+      rescue PgQuery::ParseError
+        ""
+      end
+
       # Ensures that URLS are sanitized to hide credentials
       def self.mask_url(url)
         url = url.to_s.strip

data/lib/labkit/tracing.rb

@@ -28,6 +28,12 @@ module Labkit
       ENV["GITLAB_TRACING_URL"]
     end
 
+    # Check if the current request is being traced.
+    def self.sampled?
+      context = OpenTracing.active_span&.context
+      context && context.respond_to?(:sampled?) && context.sampled?
+    end
+
     def self.stacktrace_operations
       @stacktrace_operations ||= Set.new(ENV["GITLAB_TRACING_INCLUDE_STACKTRACE"].to_s.split(",").map(&:strip))
     end

data/lib/labkit/tracing/grpc/server_interceptor.rb

@@ -14,6 +14,8 @@ module Labkit
       # for instrumenting GRPC calls with distributed tracing
       # in a GRPC Ruby server
       class ServerInterceptor < ::GRPC::ServerInterceptor
+        include Labkit::Correlation::GRPC::GRPCCommon
+
         def request_response(request: nil, call: nil, method: nil)
           wrap_with_tracing(call, method, "unary") do
             yield
@@ -40,16 +42,9 @@ module Labkit
 
         private
 
-        def route_from_method(method)
-          service_class = method.owner
-          rpc_method = method.name.to_s.split("_").map(&:capitalize).join("")
-
-          "/#{service_class.service_name}/#{rpc_method}"
-        end
-
         def wrap_with_tracing(call, method, grpc_type)
           context = TracingUtils.tracer.extract(OpenTracing::FORMAT_TEXT_MAP, call.metadata)
-          method_name = route_from_method(method)
+          method_name = "/#{rpc_split(method).join("/")}"
           tags = {
             "component" => "grpc",
             "span.kind" => "server",

data/lib/labkit/tracing/jaeger_factory.rb

@@ -19,6 +19,9 @@ module Labkit
       FLUSH_INTERVAL = 5
 
       def self.create_tracer(service_name, options)
+        # The service_name parameter from GITLAB_TRACING takes precedence over the application one
+        service_name = options[:service_name] if options[:service_name]
+
         kwargs = {
           service_name: service_name,
           sampler: get_sampler(options[:sampler], options[:sampler_param]),

data/lib/labkit/tracing/rails/active_record/sql_instrumenter.rb

@@ -14,13 +14,14 @@ module Labkit
           end
 
           def tags(payload)
+            sql = Labkit::Logging::Sanitizer.sanitize_sql(payload[:sql]) if Labkit::Tracing.sampled? && payload[:sql]
             {
               "component" => COMPONENT_TAG,
               "span.kind" => "client",
               "db.type" => "sql",
               "db.connection_id" => payload[:connection_id],
               "db.cached" => payload[:cached] || false,
-              "db.statement" => payload[:sql],
+              "db.statement" => sql,
             }
           end
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-labkit
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.13.3
 platform: ruby
 authors:
 - Andrew Newdigate
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-16 00:00:00.000000000 Z
+date: 2020-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -70,14 +70,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: opentracing
   requirement: !ruby/object:Gem::Requirement
@@ -112,6 +112,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: gitlab-pg_query
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: grpc-tools
   requirement: !ruby/object:Gem::Requirement
@@ -279,6 +293,8 @@ files:
 - lib/labkit/correlation/grpc/grpc_common.rb
 - lib/labkit/correlation/grpc/server_interceptor.rb
 - lib/labkit/logging.rb
+- lib/labkit/logging/grpc.rb
+- lib/labkit/logging/grpc/server_interceptor.rb
 - lib/labkit/logging/sanitizer.rb
 - lib/labkit/middleware.rb
 - lib/labkit/middleware/rack.rb