zuora_observability 0.1.0.pre.c → 0.1.0.pre.d

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be19d78e7a52c884e0bd5cc705a3725ea9616cd96491116d6ac047abbe8018b3
-  data.tar.gz: 446dc9c840d1d00cf66c0316d2fca0b38718d2b4ac32bcda718069515283c3f9
+  metadata.gz: f4a7c9aeb569ef2885c3f8b298771323d5a2619340b47e390d790966794b67df
+  data.tar.gz: 417163de43f269d1c5f82e2bf1c09c5066a5f752bc8e4d77d6bf27435de1d137
 SHA512:
-  metadata.gz: a70635409b8672344d7efd3c4025507ca0f450fc64aa78d5d81de9558926d1e5a59716832e7408a83abf24ed6492e4470a80fbb8bf1f84ad6391d1e188eaf94f
-  data.tar.gz: 7718bfc8650dc24092149163861a6f55ae7164cecda3f91534ee98c92645462195b9be625d0f212df974d2b87d4cc767ef9721d6f45581d47dfa0c878a53fccb
+  metadata.gz: e8435b4a4aa06bebf6d0924c8f39d71da9b96debf12582e0b92d147389e8072c8bcc40c13e2978a81cc7c8b0da9efcb540e3fa864a91e90f0b001b76d0c45667
+  data.tar.gz: a2adea8aef5fc82317077e4f68eb7061c66ca53991a545642deac1f5367cf35dfbae329fe83ee6feb3aa727e958cc0c6d7cae27c9062852affe29f8048ca6227

data/README.md

@@ -4,6 +4,8 @@ A ruby gem to enable observability into rails applications
 
 ## Usage
 
+- [Logging](doc/logging.md)
+
 ### Configuration
 
 Observability can be configured through an initializer as shown below.

data/lib/zuora_observability/configuration.rb

@@ -3,12 +3,19 @@
 module ZuoraObservability
   # Global configuration that can be set in a Rails initializer
   class Configuration
-    attr_accessor :json_logging
+    attr_reader :custom_payload_hooks
+    attr_accessor :zecs_service_hook, :json_logging
 
     def initialize
+      @custom_payload_hooks = []
+      @zecs_service_hook = nil
       # NOTE(hartley): this checks the var for presence rather than value to
       # align with how Rails does RAILS_LOG_TO_STDOUT
       @json_logging = ENV['ZUORA_JSON_LOGGING'].present? ? true : !(Rails.env.development? || Rails.env.test?)
     end
+
+    def add_custom_payload_hook(&block)
+      custom_payload_hooks << block
+    end
   end
 end

data/lib/zuora_observability/engine.rb

@@ -5,88 +5,38 @@ module ZuoraObservability
   class Engine < ::Rails::Engine
     isolate_namespace ZuoraObservability
 
-    REQUEST_HEADERS_TO_IGNORE = %w[
-      RAW_POST_DATA
-      REQUEST_METHOD
-      REQUEST_URI
-      REQUEST_PATH
-      PATH_INFO
-      CONTENT_TYPE
-      ORIGINAL_FULLPATH
-      QUERY_STRING
-    ].freeze
-
     config.generators do |g|
       g.test_framework :rspec
     end
 
     initializer(:rails_stdout_logging, before: :initialize_logger) do
-      require 'lograge'
+      Rails.application.configure do
+        config.logger = ZuoraObservability::Logger.custom_logger(name: 'Rails')
 
-      Rails.configuration.logger = ZuoraObservability::Logger.custom_logger(name: "Rails")
+        next unless ZuoraObservability.configuration.json_logging
 
-      if ZuoraObservability.configuration.json_logging
-        Rails.configuration.lograge.enabled = true
-        Rails.configuration.colorize_logging = false
-      end
+        require 'lograge'
+        require 'zuora_observability/logging/custom_options'
 
-      if Rails.configuration.lograge.enabled
-        if Rails.configuration.logger.class.to_s == 'Ougai::Logger'
-          Rails.configuration.lograge.formatter = Class.new do |fmt|
-            def fmt.call(data)
-              {
-                msg: 'Rails Request',
-                trace_id: data.delete(:trace_id),
-                zuora_trace_id: data.delete(:zuora_trace_id),
-                request: data
-              }.compact
-            end
-          end
-        end
-        #Rails.configuration.lograge.formatter = Lograge::Formatters::Json.new
-        Rails.configuration.lograge.custom_options = lambda do |event|
-          exceptions = %w(controller action format)
-          items = {
-            #time: event.time.strftime('%FT%T.%6N'),
-            params: event.payload[:params].as_json(except: exceptions).to_json.to_s,
-            trace_id: event.payload[:headers]['action_dispatch.request_id'],
-            zuora_trace_id: event.payload[:headers]['HTTP_ZUORA_REQUEST_ID']
-          }
-          items.merge!({exception_object: event.payload[:exception_object]}) if event.payload[:exception_object].present?
-          items.merge!({exception: event.payload[:exception]}) if event.payload[:exception].present?
+        config.lograge.enabled = true
+        config.colorize_logging = false
 
-          if event.payload[:headers].present?
-            # By convertion, headers usually do not have dots. Nginx even rejects headers with dots
-            # All Rails headers are namespaced, like 'rack.input'.
-            # Thus, we can obtain the client headers by rejecting dots
-            request_headers =
-              event.payload[:headers].env.
-                reject { |key| key.to_s.include?('.') || REQUEST_HEADERS_TO_IGNORE.include?(key.to_s) }
-            begin
-              if request_headers["HTTP_AUTHORIZATION"].present?
-                if request_headers["HTTP_AUTHORIZATION"].include?("Basic")
-                  user_password = request_headers["HTTP_AUTHORIZATION"].split("Basic").last.strip
-                  user, password = Base64.decode64(user_password).split(":")
-                  request_headers["HTTP_AUTHORIZATION"] = "Basic #{user}:ValueFiltered"
-                elsif
-                  request_headers["HTTP_AUTHORIZATION"] = "ValueFiltered"
-                end
-              end
-              request_headers["HTTP_API_TOKEN"] = "ValueFiltered" if request_headers["HTTP_API_TOKEN"].present?
-            rescue
-              request_headers.delete("HTTP_API_TOKEN")
-              request_headers.delete("HTTP_AUTHORIZATION")
-            end
-            items.merge!({ headers: request_headers.to_s })
-          end
+        config.lograge.formatter = Lograge::Formatters::Raw.new
 
-          if Thread.current[:appinstance].present?
-            items.merge!({connect_user: Thread.current[:appinstance].connect_user, new_session: Thread.current[:appinstance].new_session_message})
-            if Thread.current[:appinstance].logitems.present? && Thread.current[:appinstance].logitems.class == Hash
-              items.merge!(Thread.current[:appinstance].logitems)
-            end
+        config.lograge.custom_options = Logging::CustomOptions
+        config.lograge.custom_payload do |controller|
+          payload = {}
+
+          ZuoraObservability.configuration.custom_payload_hooks.each do |hook|
+            payload.merge!(hook.call(controller))
           end
-          return items
+
+          next payload unless ZuoraObservability.configuration.zecs_service_hook
+
+          payload[:zecs_service] =
+            ZuoraObservability.configuration.zecs_service_hook.call(controller)
+
+          payload
         end
       end
     end

data/lib/zuora_observability/env.rb

@@ -4,6 +4,18 @@ module ZuoraObservability
   # Methods to get information about the application environment
   class Env
     class << self
+      def name
+        ENV['Z_APPLICATION_NAME']
+      end
+
+      def version
+        ENV['Z_APPLICATION_VERSION']
+      end
+
+      def environment
+        ENV['Z_APPLICATION_ENVIRONMENT']
+      end
+
       def app_name
         # parent_name is deprecated in Rails 6.0, removed in 6.1
         ENV['DEIS_APP'].presence || Rails.application.class.parent_name

data/lib/zuora_observability/logger.rb

@@ -1,44 +1,63 @@
 # frozen_string_literal: true
 
+require 'ougai'
 require 'mono_logger'
+require 'zuora_observability/logging/formatter'
+require 'ougai/formatters/customizable'
 
 module ZuoraObservability
   # A configurable logger that can be used for Rails and additional libraries
-  module Logger
-    # NOTE(hartley): replace this with a subclass of Ougai Logger
-    # - enable silence method for Rails 5.2 asset requests
+  class Logger < Ougai::Logger
+    # TODO(hartley): enable silence method for Rails 5.2 asset requests
     # https://github.com/tilfin/ougai/wiki/Use-as-Rails-logger
-    # - potentially integrate with MonoLogger so no conditional is needed
-    # https://github.com/tilfin/ougai/issues/74
-    def self.custom_logger(name: "", level: Rails.logger.present? ? Rails.logger.level : MonoLogger::INFO, type: :ougai)
-      #puts name + ' - ' + {Logger::WARN => 'Logger::WARN', Logger::ERROR => 'Logger::ERROR', Logger::DEBUG => 'Logger::DEBUG', Logger::INFO => 'Logger::INFO' }[level] + ' - '
+    def initialize(logdev, **)
+      super
+
+      # NOTE(hartley): the purpose for the original split between Ougai and
+      # MonoLogger was that MonoLogger enables logging in a trap context
+      # https://github.com/tilfin/ougai/issues/74
+      # By using our own Logger class, we can override the LogDevice created
+      # by ruby with MonoLogger's, enabling logging in a trap context
+      @logdev = MonoLogger::LocklessLogDevice.new(logdev)
+    end
+
+    def create_formatter
+      return ZuoraObservability::Logging::Formatter.new if ZuoraObservability.configuration.json_logging
+
+      formatter = Ougai::Formatters::Customizable.new(
+        format_err: method(:custom_error_formatter),
+        format_data: method(:custom_data_formatter),
+        format_msg: method(:custom_message_formatter)
+      )
+      formatter.datetime_format = '%FT%T.%3NZ'
+      formatter
+    end
+
+    def custom_message_formatter(severity, datetime, _progname, data)
+      msg = data.delete(:msg)
+      "#{severity.ljust(6)} #{datetime}: #{msg}"
+    end
+
+    def custom_data_formatter(data)
+      data.delete_if do |k|
+        %i[app_instance_id tenant_ids organization environment].include? k
+      end
+
+      return nil if data.blank?
+
+      "#{'DATA'.ljust(6)} #{Time.current.strftime('%FT%T.%3NZ')}: #{data.to_json}"
+    end
+
+    def custom_error_formatter(data)
+      return nil unless data.key?(:err)
+
+      err = data.delete(:err)
+      "  #{err[:name]} (#{err[:message]})\n #{err[:stack]}"
+    end
+
+    def self.custom_logger(name: '', level: Rails.logger.present? ? Rails.logger.level : MonoLogger::INFO, type: :ougai)
       if type == :ougai
-        require 'ougai'
-        require "ougai/formatters/customizable"
-        #logger = Ougai::Logger.new(MonoLogger.new(STDOUT))
-        logger = Ougai::Logger.new(STDOUT)
-        logger.level = level
-        if ZuoraObservability.configuration.json_logging
-          require 'zuora_observability/logging/formatter'
-          logger.formatter = ZuoraObservability::Logging::Formatter.new(name)
-        else
-          logger.formatter = Ougai::Formatters::Customizable.new(
-            format_err: proc do |data|
-              next nil unless data.key?(:err)
-              err = data.delete(:err)
-              "  #{err[:name]} (#{err[:message]})\n #{err[:stack]}"
-            end,
-            format_data: proc do |data|
-              data.delete(:app_instance_id); data.delete(:tenant_ids); data.delete(:organization); data.delete(:environment)
-              format('%s %s: %s', 'DATA'.ljust(6), Time.now.strftime('%FT%T.%6NZ'), "#{data.to_json}") if data.present?
-            end,
-            format_msg: proc do |severity, datetime, _progname, data|
-              msg = data.delete(:msg)
-              format('%s %s: %s', severity.ljust(6), datetime, msg)
-            end
-          )
-          logger.formatter.datetime_format = '%FT%T.%6NZ'
-        end
+        logger = new($stdout, level: level, progname: name)
       else
         require 'mono_logger'
         logger = MonoLogger.new(STDOUT)
@@ -63,7 +82,8 @@ module ZuoraObservability
           end
         end
       end
-      return logger
+
+      logger
     end
   end
 end

data/lib/zuora_observability/logging/custom_options.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module ZuoraObservability
+  module Logging
+    # Custom Options for lograge to add additional fields to logged data
+    module CustomOptions
+      IGNORE_HEADERS = %w[
+        RAW_POST_DATA
+        REQUEST_METHOD
+        REQUEST_URI
+        REQUEST_PATH
+        PATH_INFO
+        CONTENT_TYPE
+        ORIGINAL_FULLPATH
+        QUERY_STRING
+      ].freeze
+
+      IGNORE_PARAMS = %w[controller action format].freeze
+
+      FILTERED = '[FILTERED]'
+
+      class << self
+        def call(event)
+          items = {
+            msg: 'Rails Request',
+            params: event.payload[:params].as_json(except: IGNORE_PARAMS).to_s,
+            trace_id: event.payload[:headers]['action_dispatch.request_id'],
+            zuora_trace_id: event.payload[:headers]['HTTP_ZUORA_REQUEST_ID'],
+            error: event.payload[:exception_object]
+          }
+
+          if event.payload[:headers].present?
+            request_headers = filter_request_headers(event.payload[:headers])
+
+            filtered_headers = filter_sensitive_headers(request_headers)
+
+            items.merge!({ headers: filtered_headers })
+          end
+
+          if Thread.current[:appinstance].present?
+            items.merge!({connect_user: Thread.current[:appinstance].connect_user, new_session: Thread.current[:appinstance].new_session_message})
+            if Thread.current[:appinstance].logitems.present? && Thread.current[:appinstance].logitems.class == Hash
+              items.merge!(Thread.current[:appinstance].logitems)
+            end
+          end
+
+          items
+        end
+
+        private
+
+        def filter_request_headers(action_dispatch_headers)
+          action_dispatch_headers.env.select do |header|
+            next false if IGNORE_HEADERS.include? header
+
+            header =~ /^HTTP_/ || ActionDispatch::Http::Headers::CGI_VARIABLES.include?(header)
+          end
+        end
+
+        def filter_sensitive_headers(headers)
+          filtered = {}
+
+          auth = headers['HTTP_AUTHORIZATION']
+          filtered['HTTP_AUTHORIZATION'] = filter_auth_header(auth) if auth
+
+          filtered['HTTP_API_TOKEN'] = FILTERED if headers['HTTP_API_TOKEN']
+
+          headers.merge(filtered)
+        end
+
+        def filter_auth_header(authorization)
+          return FILTERED unless authorization.include? 'Basic'
+
+          encoded = authorization.split(' ', 2).second
+          user, _password = Base64.decode64(encoded).split(':')
+
+          "Basic #{user}:#{FILTERED}"
+        end
+      end
+    end
+  end
+end

data/lib/zuora_observability/logging/formatter.rb

@@ -1,45 +1,169 @@
 # frozen_string_literal: true
 
-require 'ougai/formatters/base'
-require 'ougai/formatters/for_json'
-
 module ZuoraObservability
   module Logging
-    # A JSON formatter compatible with node-bunyan
-    class Formatter < Ougai::Formatters::Base
-      include Ougai::Formatters::ForJson
-
-      # Intialize a formatter
-      # @param [String] app_name application name (execution program name if nil)
-      # @param [String] hostname hostname (hostname if nil)
-      # @param [Hash] opts the initial values of attributes
-      # @option opts [String] :trace_indent (2) the value of trace_indent attribute
-      # @option opts [String] :trace_max_lines (100) the value of trace_max_lines attribute
-      # @option opts [String] :serialize_backtrace (true) the value of serialize_backtrace attribute
-      # @option opts [String] :jsonize (true) the value of jsonize attribute
-      # @option opts [String] :with_newline (true) the value of with_newline attribute
-      def initialize(app_name = nil, hostname = nil, opts = {})
-        aname, hname, opts = Ougai::Formatters::Base.parse_new_params([app_name, hostname, opts])
-        super(aname, hname, opts)
-        init_opts_for_json(opts)
-      end
+    # Formats data into ZECS Logging Standard and dumps as JSON
+    # https://tag.pages.gitlab.zeta.tools/standards/logging
+    class Formatter < Ougai::Formatters::Bunyan
+      ECS_VERSION = '1.5.0'
+
+      ZECS_TOP_LEVEL_FIELDS = %i[
+        error event http organization process url user
+      ].freeze
+      ECS_HTTP_FIELDS = %i[request response].freeze
 
       def _call(severity, time, progname, data)
-        data.merge!({ message: data.delete(:msg) })
-        if data[:timestamp].present?
-          time = data[:timestamp]
-          data.delete(:timestamp)
-        end
-        dump({
-          name: progname || @app_name,
-          pid: $$,
-          level: severity,
-          timestamp: time.utc.strftime('%FT%T.%6NZ'),
-        }.merge(data))
+        output = base_fields(severity, time, progname, data)
+
+        other_fields = get_fields_for(ZECS_TOP_LEVEL_FIELDS, data)
+        output.merge!(other_fields)
+
+        zuora_fields = ZuoraFields.call(data)
+        output[:zuora] = zuora_fields unless zuora_fields.empty?
+
+        dump(output)
       end
 
+      # Note(hartley): Ougai::Formatters::ForJson requires this be present
       def convert_time(data)
-        # data[:timestamp] = format_datetime(data[:time])
+        data[:@timestamp] = data[:@timestamp].utc.iso8601(3)
+      end
+
+      private
+
+      def get_fields_for(list, data)
+        final_hash = {}
+
+        list.each do |field|
+          field_output = send("#{field}_fields", data)
+          final_hash[field] = field_output unless field_output.empty?
+        end
+
+        final_hash
+      end
+
+      # Fields required by ECS
+      def base_fields(severity, time, progname, data)
+        {
+          :@timestamp => time,
+          message: data[:msg],
+          ecs: { version: ECS_VERSION },
+          log: { level: severity, logger: progname || @app_name },
+          service: { name: Env.name, version: Env.version },
+          trace: { id: data[:trace_id] }
+        }
+      end
+
+      # error
+      def error_fields(data)
+        return {} unless data[:error]
+
+        {
+          message: data[:error].message,
+          stack_trace: data[:error].backtrace.join("\n"),
+          type: data[:error].class
+        }.compact
+      end
+
+      # http
+      def http_fields(data)
+        get_fields_for(ECS_HTTP_FIELDS, data)
+      end
+
+      # http.request
+      def request_fields(data)
+        {
+          method: data[:method],
+          body: ({ content: data[:params] } if data.key? :params)
+        }.compact
+      end
+
+      # http.response
+      def response_fields(data)
+        {
+          status_code: data[:status]
+        }.compact
+      end
+
+      # organization
+      def organization_fields(data)
+        {
+          name: data[:organization]
+        }.compact
+      end
+
+      # process
+      def process_fields(_data)
+        { id: Process.pid }
+      end
+
+      # url
+      def url_fields(data)
+        {
+          path: data[:path]
+        }.compact
+      end
+
+      # user
+      def user_fields(data)
+        {
+          email: data[:email]
+        }.compact
+      end
+
+      # event
+      def event_fields(data)
+        full_action = "#{data[:controller]}##{data[:action]}"
+
+        {
+          action: (full_action unless full_action == '#'),
+          duration: data[:duration]
+        }.compact
+      end
+    end
+
+    # The Zuora Extension to ECS defined by ZECS
+    module ZuoraFields
+      ZECS_VERSION = '1.0'
+
+      class << self
+        # zuora top level field
+        def call(data)
+          hash = base_fields(data)
+
+          z_http = http_fields(data)
+          hash[:http] = z_http unless z_http.empty?
+
+          z_service = service_fields(data)
+          hash[Env.name.to_sym] = z_service unless z_service.empty?
+
+          return {} if hash.empty?
+
+          hash.merge({ ecs: { version: ZECS_VERSION } })
+        end
+
+        def base_fields(data)
+          {
+            apartment_id: data[:app_instance_id],
+            environment: data[:environment],
+            tenant_id: data[:tenant_ids],
+            trace_id: data[:zuora_trace_id]
+          }.compact
+        end
+
+        # zuora.http
+        def http_fields(data)
+          z_http_request = { headers: data[:headers] }.compact
+
+          {
+            request: (z_http_request unless z_http_request.empty?)
+          }.compact
+        end
+
+        # Service's Custom Fields
+        def service_fields(data)
+          data[:zecs_service].presence || {}
+        end
       end
     end
   end

data/lib/zuora_observability/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ZuoraObservability
-  VERSION = '0.1.0-c'
+  VERSION = '0.1.0-d'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zuora_observability
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.c
+  version: 0.1.0.pre.d
 platform: ruby
 authors:
 - Hartley McGuire
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-05 00:00:00.000000000 Z
+date: 2021-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: lograge
@@ -261,6 +261,7 @@ files:
 - lib/zuora_observability/engine.rb
 - lib/zuora_observability/env.rb
 - lib/zuora_observability/logger.rb
+- lib/zuora_observability/logging/custom_options.rb
 - lib/zuora_observability/logging/formatter.rb
 - lib/zuora_observability/metrics.rb
 - lib/zuora_observability/version.rb
@@ -276,7 +277,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">"