zuora_observability 0.1.0.pre.b → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3c6f15e5345dd32d8ada993579cf90efa2a2382573fe86f608f598be3f69f9e
-  data.tar.gz: 38035565943c737e2f5ac59032780b6ce0c9e2eed0d039870d9b31b4780f7629
+  metadata.gz: 52240e5716643d75479140930fd29ed88b154af9c01511fcceea92338b399ae2
+  data.tar.gz: 20fa0460d7e665ee83b6265f69d928ed74b90fef9231e7211dac287d0e0475f0
 SHA512:
-  metadata.gz: c51373a4c25bebde5158c57e035a4363895f67c6f991fc7dbcd9a0741496e65ae07d7f6f65da86f8de14c4ef86ea509f0ba110d9fd7e3d29dcaa83e19117bbd6
-  data.tar.gz: ca248b0270c959804ded1396591e6bacde2abaa2f478ea3b12c2a8265716562ddf0f8f87ddbd90f1ba228657568c337258936faaccebb9540dcbdff59e2903f9
+  metadata.gz: 0246f0a9836504bbd5fb8ed0e0ec909664a6848cca592a4756c47dff3153eeb763e3f7e179fe031fe65161b9bf01ffbbc396f995738501f625dcbdf9b824436e
+  data.tar.gz: b7da319f502643521f8b229e2f33b889d9fbb6823b38b948b51d38f073f348d59affc8f84fca734635294627225766ab1ea7f2d49ac91b7b71a71c2b9e396518

data/README.md

@@ -2,25 +2,10 @@
 
 A ruby gem to enable observability into rails applications
 
-## Usage
-
-### Configuration
-
-Observability can be configured through an initializer as shown below.
+## Prerequisites
 
-```ruby
-# config/initializers/zuora_observability.rb
-ZuoraObservability.configure do |config|
-  config.enable_metrics = true
-end
-```
-
-| Configuration Field | Default Value                                                    | Description                          |
-| ------------------- | ---------------------------------------------------------------- | ------------------------------------ |
-| `enable_metrics`    | `false`                                                          | Enables writing metrics to telegraf  |
-| `telegraf_endpoint` | `'udp://telegraf-app-metrics.monitoring.svc.cluster.local:8094'` | The URL to send telegraf metrics to  |
-| `telegraf_debug`    | `false`                                                          | Log info about data sent to telegraf |
-| `json_logging`      | `false` in development/test, `true` otherwise                    | Use structured JSON logging for ZuoraObservability::Logger. **This configuration option cannot be changed in an initializer, but can be set to `true` with the presence of `ZUORA_JSON_LOGGING` environment variable** |
+- Ruby 2.5+
+- Rails 5.0+
 
 ## Installation
 
@@ -42,6 +27,10 @@ Or install it yourself as:
 $ gem install zuora_observability
 ```
 
+## Usage
+
+- [Logging](doc/logging.md)
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/config/initializers/loggers.rb

@@ -1,16 +1,14 @@
 # frozen_string_literal: true
 
 if defined?(Resque.logger)
-  Resque.logger = ZuoraObservability::Logger.custom_logger(name: 'Resque', type: 'Monologger', level: MonoLogger::INFO)
+  Resque.logger = ZuoraObservability::Logger.custom_logger(name: 'Resque')
   if defined?(Resque::Scheduler)
     Resque::Scheduler.logger = ZuoraObservability::Logger.custom_logger(name: 'ResqueScheduler')
   end
 end
 
 if defined?(Delayed::Worker.logger)
-  Delayed::Worker.logger = ZuoraObservability::Logger.custom_logger(
-    name: 'DelayedJob', type: 'Monologger', level: MonoLogger::INFO
-  )
+  Delayed::Worker.logger = ZuoraObservability::Logger.custom_logger(name: 'DelayedJob', level: Logger::INFO)
 end
 
 if defined?(Makara)
@@ -18,9 +16,9 @@ if defined?(Makara)
 end
 
 if defined?(ElasticAPM) && ElasticAPM.running?
-  ElasticAPM.agent.config.logger = ZuoraObservability::Logger.custom_logger(name: 'ElasticAPM', level: MonoLogger::WARN)
+  ElasticAPM.agent.config.logger = ZuoraObservability::Logger.custom_logger(name: 'ElasticAPM', level: Logger::WARN)
 end
 
 if defined?(ActionMailer)
-  ActionMailer::Base.logger = ZuoraObservability::Logger.custom_logger(name: 'ActionMailer', type: 'Monologger')
+  ActionMailer::Base.logger = ZuoraObservability::Logger.custom_logger(name: 'ActionMailer')
 end

data/lib/zuora_observability.rb

@@ -7,8 +7,6 @@ require 'zuora_observability/metrics'
 require 'zuora_observability/logger'
 
 require 'zuora_observability/logging/formatter'
-require 'zuora_observability/metrics/telegraf'
-require 'zuora_observability/metrics/point_value'
 
 # Provides Rails application with tools for observabilty
 module ZuoraObservability

data/lib/zuora_observability/configuration.rb

@@ -3,16 +3,19 @@
 module ZuoraObservability
   # Global configuration that can be set in a Rails initializer
   class Configuration
-    attr_accessor :enable_metrics, :telegraf_endpoint, :telegraf_debug,
-                  :json_logging
+    attr_reader :custom_payload_hooks
+    attr_accessor :zecs_service_hook, :json_logging
 
     def initialize
-      @enable_metrics = false
-      @telegraf_endpoint = 'udp://telegraf-app-metrics.monitoring.svc.cluster.local:8094'
-      @telegraf_debug = false
+      @custom_payload_hooks = []
+      @zecs_service_hook = nil
       # NOTE(hartley): this checks the var for presence rather than value to
       # align with how Rails does RAILS_LOG_TO_STDOUT
       @json_logging = ENV['ZUORA_JSON_LOGGING'].present? ? true : !(Rails.env.development? || Rails.env.test?)
     end
+
+    def add_custom_payload_hook(&block)
+      custom_payload_hooks << block
+    end
   end
 end

data/lib/zuora_observability/engine.rb

@@ -1,93 +1,42 @@
 # frozen_string_literal: true
 
-require 'zuora_observability/metrics/page_request'
-
 module ZuoraObservability
   # The ZuoraObservability Engine is mounted to hook into Rails
   class Engine < ::Rails::Engine
     isolate_namespace ZuoraObservability
 
-    REQUEST_HEADERS_TO_IGNORE = %w[
-      RAW_POST_DATA
-      REQUEST_METHOD
-      REQUEST_URI
-      REQUEST_PATH
-      PATH_INFO
-      CONTENT_TYPE
-      ORIGINAL_FULLPATH
-      QUERY_STRING
-    ].freeze
-
     config.generators do |g|
       g.test_framework :rspec
     end
 
-    # hook to process_action
-    ActiveSupport::Notifications.subscribe(
-      'process_action.action_controller',
-      ZuoraObservability::Metrics::PageRequest.new
-    )
-
     initializer(:rails_stdout_logging, before: :initialize_logger) do
-      require 'lograge'
+      Rails.application.configure do
+        config.logger = ZuoraObservability::Logger.custom_logger(name: 'Rails')
 
-      Rails.configuration.logger = ZuoraObservability::Logger.custom_logger(name: "Rails")
+        next unless ZuoraObservability.configuration.json_logging
 
-      if ZuoraObservability.configuration.json_logging
-        Rails.configuration.lograge.enabled = true
-        Rails.configuration.colorize_logging = false
-      end
+        require 'lograge'
+        require 'zuora_observability/logging/custom_options'
 
-      if Rails.configuration.lograge.enabled
-        if Rails.configuration.logger.class.to_s == 'Ougai::Logger'
-          Rails.configuration.lograge.formatter = Class.new do |fmt|
-            def fmt.call(data)
-              { msg: 'Rails Request', request: data }
-            end
-          end
-        end
-        #Rails.configuration.lograge.formatter = Lograge::Formatters::Json.new
-        Rails.configuration.lograge.custom_options = lambda do |event|
-          exceptions = %w(controller action format)
-          items = {
-            #time: event.time.strftime('%FT%T.%6N'),
-            params: event.payload[:params].as_json(except: exceptions).to_json.to_s
-          }
-          items.merge!({exception_object: event.payload[:exception_object]}) if event.payload[:exception_object].present?
-          items.merge!({exception: event.payload[:exception]}) if event.payload[:exception].present?
+        config.lograge.enabled = true
+        config.colorize_logging = false
 
-          if event.payload[:headers].present?
-            # By convertion, headers usually do not have dots. Nginx even rejects headers with dots
-            # All Rails headers are namespaced, like 'rack.input'.
-            # Thus, we can obtain the client headers by rejecting dots
-            request_headers =
-              event.payload[:headers].env.
-                reject { |key| key.to_s.include?('.') || REQUEST_HEADERS_TO_IGNORE.include?(key.to_s) }
-            begin
-              if request_headers["HTTP_AUTHORIZATION"].present?
-                if request_headers["HTTP_AUTHORIZATION"].include?("Basic")
-                  user_password = request_headers["HTTP_AUTHORIZATION"].split("Basic").last.strip
-                  user, password = Base64.decode64(user_password).split(":")
-                  request_headers["HTTP_AUTHORIZATION"] = "Basic #{user}:ValueFiltered"
-                elsif
-                  request_headers["HTTP_AUTHORIZATION"] = "ValueFiltered"
-                end
-              end
-              request_headers["HTTP_API_TOKEN"] = "ValueFiltered" if request_headers["HTTP_API_TOKEN"].present?
-            rescue
-              request_headers.delete("HTTP_API_TOKEN")
-              request_headers.delete("HTTP_AUTHORIZATION")
-            end
-            items.merge!({ headers: request_headers.to_s })
-          end
+        config.lograge.formatter = Lograge::Formatters::Raw.new
+
+        config.lograge.custom_options = Logging::CustomOptions
+        config.lograge.custom_payload do |controller|
+          payload = {}
 
-          if Thread.current[:appinstance].present?
-            items.merge!({connect_user: Thread.current[:appinstance].connect_user, new_session: Thread.current[:appinstance].new_session_message})
-            if Thread.current[:appinstance].logitems.present? && Thread.current[:appinstance].logitems.class == Hash
-              items.merge!(Thread.current[:appinstance].logitems)
-            end
+          ZuoraObservability.configuration.custom_payload_hooks.each do |hook|
+            payload.merge!(hook.call(controller))
           end
-          return items
+
+          next payload unless ZuoraObservability.configuration.zecs_service_hook
+
+          payload[:zecs_service] =
+            ZuoraObservability.configuration.zecs_service_hook.call(controller)
+
+          payload
         end
       end
     end

data/lib/zuora_observability/env.rb

@@ -4,9 +4,20 @@ module ZuoraObservability
   # Methods to get information about the application environment
   class Env
     class << self
+      def name
+        ENV['Z_APPLICATION_NAME']
+      end
+
+      def version
+        ENV['Z_APPLICATION_VERSION']
+      end
+
+      def environment
+        ENV['Z_APPLICATION_ENVIRONMENT']
+      end
+
       def app_name
-        # parent_name is deprecated in Rails 6.0, removed in 6.1
-        ENV['DEIS_APP'].presence || Rails.application.class.parent_name
+        ENV['DEIS_APP'].presence || app_parent_name
       end
 
       def pod_name
@@ -28,6 +39,16 @@ module ZuoraObservability
         end
         return p_type
       end
+
+      private
+
+      def app_parent_name
+        if Rails::VERSION::MAJOR >= 6 && Rails::VERSION::MINOR >= 1
+          Rails.application.class.module_parent_name
+        else
+          Rails.application.class.parent_name
+        end
+      end
     end
   end
 end

data/lib/zuora_observability/logger.rb

@@ -1,49 +1,63 @@
 # frozen_string_literal: true
 
+require 'ougai'
 require 'mono_logger'
+require 'zuora_observability/logging/formatter'
+require 'ougai/formatters/customizable'
 
 module ZuoraObservability
   # A configurable logger that can be used for Rails and additional libraries
-  module Logger
-    # NOTE(hartley): replace this with a subclass of Ougai Logger
-    # - enable silence method for Rails 5.2 asset requests
+  class Logger < Ougai::Logger
+    # TODO(hartley): enable silence method for Rails 5.2 asset requests
     # https://github.com/tilfin/ougai/wiki/Use-as-Rails-logger
-    # - potentially integrate with MonoLogger so no conditional is needed
-    # https://github.com/tilfin/ougai/issues/74
-    def self.custom_logger(name: "", level: Rails.logger.present? ? Rails.logger.level : MonoLogger::INFO, type: :ougai)
-      #puts name + ' - ' + {Logger::WARN => 'Logger::WARN', Logger::ERROR => 'Logger::ERROR', Logger::DEBUG => 'Logger::DEBUG', Logger::INFO => 'Logger::INFO' }[level] + ' - '
+    def initialize(logdev, **)
+      super
+
+      # NOTE(hartley): the purpose for the original split between Ougai and
+      # MonoLogger was that MonoLogger enables logging in a trap context
+      # https://github.com/tilfin/ougai/issues/74
+      # By using our own Logger class, we can override the LogDevice created
+      # by ruby with MonoLogger's, enabling logging in a trap context
+      @logdev = MonoLogger::LocklessLogDevice.new(logdev)
+    end
+
+    def create_formatter
+      return ZuoraObservability::Logging::Formatter.new if ZuoraObservability.configuration.json_logging
+
+      formatter = Ougai::Formatters::Customizable.new(
+        format_err: method(:custom_error_formatter),
+        format_data: method(:custom_data_formatter),
+        format_msg: method(:custom_message_formatter)
+      )
+      formatter.datetime_format = '%FT%T.%3NZ'
+      formatter
+    end
+
+    def custom_message_formatter(severity, datetime, _progname, data)
+      msg = data.delete(:msg)
+      "#{severity.ljust(6)} #{datetime}: #{msg}"
+    end
+
+    def custom_data_formatter(data)
+      data.delete_if do |k|
+        %i[app_instance_id tenant_ids organization environment].include? k
+      end
+
+      return nil if data.blank?
+
+      "#{'DATA'.ljust(6)} #{Time.current.strftime('%FT%T.%3NZ')}: #{data.to_json}"
+    end
+
+    def custom_error_formatter(data)
+      return nil unless data.key?(:err)
+
+      err = data.delete(:err)
+      "  #{err[:name]} (#{err[:message]})\n #{err[:stack]}"
+    end
+
+    def self.custom_logger(name: '', level: Rails.logger.present? ? Rails.logger.level : MonoLogger::INFO, type: :ougai)
       if type == :ougai
-        require 'ougai'
-        require "ougai/formatters/customizable"
-        #logger = Ougai::Logger.new(MonoLogger.new(STDOUT))
-        logger = Ougai::Logger.new(STDOUT)
-        logger.level = level
-        if ZuoraObservability.configuration.json_logging
-          require 'zuora_observability/logging/formatter'
-          logger.formatter = ZuoraObservability::Logging::Formatter.new(name)
-          logger.before_log = lambda do |data|
-            data[:trace_id] = ZuoraConnect::RequestIdMiddleware.request_id if ZuoraConnect::RequestIdMiddleware.request_id.present?
-            data[:zuora_trace_id] = ZuoraConnect::RequestIdMiddleware.zuora_request_id if ZuoraConnect::RequestIdMiddleware.zuora_request_id.present?
-            #data[:traces] = {amazon_id: data[:trace_id], zuora_id: data[:zuora_trace_id]}
-          end
-        else
-          logger.formatter = Ougai::Formatters::Customizable.new(
-            format_err: proc do |data|
-              next nil unless data.key?(:err)
-              err = data.delete(:err)
-              "  #{err[:name]} (#{err[:message]})\n #{err[:stack]}"
-            end,
-            format_data: proc do |data|
-              data.delete(:app_instance_id); data.delete(:tenant_ids); data.delete(:organization); data.delete(:environment)
-              format('%s %s: %s', 'DATA'.ljust(6), Time.now.strftime('%FT%T.%6NZ'), "#{data.to_json}") if data.present?
-            end,
-            format_msg: proc do |severity, datetime, _progname, data|
-              msg = data.delete(:msg)
-              format('%s %s: %s', severity.ljust(6), datetime, msg)
-            end
-          )
-          logger.formatter.datetime_format = '%FT%T.%6NZ'
-        end
+        logger = new($stdout, level: level, progname: name)
       else
         require 'mono_logger'
         logger = MonoLogger.new(STDOUT)
@@ -68,7 +82,8 @@ module ZuoraObservability
           end
         end
       end
-      return logger
+
+      logger
     end
   end
 end

data/lib/zuora_observability/logging/custom_options.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module ZuoraObservability
+  module Logging
+    # Custom Options for lograge to add additional fields to logged data
+    module CustomOptions
+      IGNORE_HEADERS = %w[
+        RAW_POST_DATA
+        REQUEST_METHOD
+        REQUEST_URI
+        REQUEST_PATH
+        PATH_INFO
+        CONTENT_TYPE
+        ORIGINAL_FULLPATH
+        QUERY_STRING
+      ].freeze
+
+      IGNORE_PARAMS = %w[controller action format].freeze
+
+      FILTERED = '[FILTERED]'
+
+      class << self
+        def call(event)
+          items = {
+            msg: 'Rails Request',
+            params: event.payload[:params].as_json(except: IGNORE_PARAMS).to_s,
+            trace_id: event.payload[:headers]['action_dispatch.request_id'],
+            zuora_trace_id: event.payload[:headers]['HTTP_ZUORA_REQUEST_ID'],
+            error: event.payload[:exception_object]
+          }
+
+          if event.payload[:headers].present?
+            request_headers = filter_request_headers(event.payload[:headers])
+
+            filtered_headers = filter_sensitive_headers(request_headers)
+
+            items.merge!({ headers: filtered_headers })
+          end
+
+          items
+        end
+
+        private
+
+        def filter_request_headers(action_dispatch_headers)
+          action_dispatch_headers.env.select do |header|
+            next false if IGNORE_HEADERS.include? header
+
+            header =~ /^HTTP_/ || ActionDispatch::Http::Headers::CGI_VARIABLES.include?(header)
+          end
+        end
+
+        def filter_sensitive_headers(headers)
+          filtered = {}
+
+          auth = headers['HTTP_AUTHORIZATION']
+          filtered['HTTP_AUTHORIZATION'] = filter_auth_header(auth) if auth
+
+          filtered['HTTP_API_TOKEN'] = FILTERED if headers['HTTP_API_TOKEN']
+
+          headers.merge(filtered)
+        end
+
+        def filter_auth_header(authorization)
+          return FILTERED unless authorization.include? 'Basic'
+
+          encoded = authorization.split(' ', 2).second
+          user, _password = Base64.decode64(encoded).split(':')
+
+          "Basic #{user}:#{FILTERED}"
+        end
+      end
+    end
+  end
+end

data/lib/zuora_observability/logging/formatter.rb

@@ -1,45 +1,170 @@
 # frozen_string_literal: true
 
-require 'ougai/formatters/base'
-require 'ougai/formatters/for_json'
-
 module ZuoraObservability
   module Logging
-    # A JSON formatter compatible with node-bunyan
-    class Formatter < Ougai::Formatters::Base
-      include Ougai::Formatters::ForJson
-
-      # Intialize a formatter
-      # @param [String] app_name application name (execution program name if nil)
-      # @param [String] hostname hostname (hostname if nil)
-      # @param [Hash] opts the initial values of attributes
-      # @option opts [String] :trace_indent (2) the value of trace_indent attribute
-      # @option opts [String] :trace_max_lines (100) the value of trace_max_lines attribute
-      # @option opts [String] :serialize_backtrace (true) the value of serialize_backtrace attribute
-      # @option opts [String] :jsonize (true) the value of jsonize attribute
-      # @option opts [String] :with_newline (true) the value of with_newline attribute
-      def initialize(app_name = nil, hostname = nil, opts = {})
-        aname, hname, opts = Ougai::Formatters::Base.parse_new_params([app_name, hostname, opts])
-        super(aname, hname, opts)
-        init_opts_for_json(opts)
-      end
+    # Formats data into ZECS Logging Standard and dumps as JSON
+    # https://tag.pages.gitlab.zeta.tools/standards/logging
+    class Formatter < Ougai::Formatters::Bunyan
+      ECS_VERSION = '1.5.0'
+
+      ZECS_TOP_LEVEL_FIELDS = %i[
+        error event http organization process url user
+      ].freeze
+      ECS_HTTP_FIELDS = %i[request response].freeze
 
       def _call(severity, time, progname, data)
-        data.merge!({ message: data.delete(:msg) })
-        if data[:timestamp].present?
-          time = data[:timestamp]
-          data.delete(:timestamp)
-        end
-        dump({
-          name: progname || @app_name,
-          pid: $$,
-          level: severity,
-          timestamp: time.utc.strftime('%FT%T.%6NZ'),
-        }.merge(data))
+        output = base_fields(severity, time, progname, data)
+
+        other_fields = get_fields_for(ZECS_TOP_LEVEL_FIELDS, data)
+        output.merge!(other_fields)
+
+        zuora_fields = ZuoraFields.call(data)
+        output[:zuora] = zuora_fields unless zuora_fields.empty?
+
+        dump(output)
       end
 
+      # Note(hartley): Ougai::Formatters::ForJson requires this be present
       def convert_time(data)
-        # data[:timestamp] = format_datetime(data[:time])
+        data[:@timestamp] = data[:@timestamp].utc.iso8601(3)
+      end
+
+      private
+
+      def get_fields_for(list, data)
+        final_hash = {}
+
+        list.each do |field|
+          field_output = send("#{field}_fields", data)
+          final_hash[field] = field_output unless field_output.empty?
+        end
+
+        final_hash
+      end
+
+      # Fields required by ECS
+      def base_fields(severity, time, progname, data)
+        {
+          :@timestamp => time,
+          message: data[:msg],
+          ecs: { version: ECS_VERSION },
+          log: { level: severity, logger: progname || @app_name },
+          service: { name: Env.name, version: Env.version },
+          trace: { id: data[:trace_id] }
+        }
+      end
+
+      # error
+      def error_fields(data)
+        return {} unless data[:error]
+
+        {
+          message: data[:error].message,
+          stack_trace: data[:error].backtrace.join("\n"),
+          type: data[:error].class
+        }.compact
+      end
+
+      # http
+      def http_fields(data)
+        get_fields_for(ECS_HTTP_FIELDS, data)
+      end
+
+      # http.request
+      def request_fields(data)
+        {
+          method: data[:method],
+          body: ({ content: data[:params] } if data.key? :params)
+        }.compact
+      end
+
+      # http.response
+      def response_fields(data)
+        {
+          status_code: data[:status]
+        }.compact
+      end
+
+      # organization
+      def organization_fields(data)
+        {
+          name: data[:organization]
+        }.compact
+      end
+
+      # process
+      def process_fields(_data)
+        { id: Process.pid }
+      end
+
+      # url
+      def url_fields(data)
+        {
+          path: data[:path]
+        }.compact
+      end
+
+      # user
+      def user_fields(data)
+        {
+          email: data[:email]
+        }.compact
+      end
+
+      # event
+      def event_fields(data)
+        full_action = "#{data[:controller]}##{data[:action]}"
+
+        {
+          action: (full_action unless full_action == '#'),
+          duration: data[:duration]
+        }.compact
+      end
+    end
+
+    # The Zuora Extension to ECS defined by ZECS
+    module ZuoraFields
+      ZECS_VERSION = '1.1'
+
+      class << self
+        # zuora top level field
+        def call(data)
+          hash = base_fields(data)
+
+          z_http = http_fields(data)
+          hash[:http] = z_http unless z_http.empty?
+
+          z_service = service_fields(data)
+          hash[Env.name.to_sym] = z_service unless z_service.empty?
+
+          return {} if hash.empty?
+
+          hash.merge({ ecs: { version: ZECS_VERSION } })
+        end
+
+        def base_fields(data)
+          {
+            apartment_id: data[:app_instance_id],
+            cp_id: data[:zuora_track_id],
+            environment: data[:environment],
+            tenant_id: data[:tenant_ids],
+            trace_id: data[:zuora_trace_id]
+          }.compact
+        end
+
+        # zuora.http
+        def http_fields(data)
+          z_http_request = { headers: data[:headers] }.compact
+
+          {
+            request: (z_http_request unless z_http_request.empty?)
+          }.compact
+        end
+
+        # Service's Custom Fields
+        def service_fields(data)
+          data[:zecs_service].presence || {}
+        end
       end
     end
   end

data/lib/zuora_observability/metrics.rb

@@ -3,18 +3,7 @@
 module ZuoraObservability
   # Methods to gather and format metrics
   module Metrics
-    @@telegraf_host = nil
-
     class << self
-      def write_to_telegraf(*args)
-        if ZuoraObservability.configuration.enable_metrics && !defined?(Prometheus)
-          @@telegraf_host = Metrics::Telegraf.new() if @@telegraf_host == nil
-          unicorn_stats = Metrics.unicorn_listener if defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
-          @@telegraf_host.write(direction: 'Raindrops', tags: {}, values: unicorn_stats)  unless unicorn_stats.blank?
-          return @@telegraf_host.write(*args)
-        end
-      end
-
       def resque
         Resque.redis.ping
 

data/lib/zuora_observability/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ZuoraObservability
-  VERSION = '0.1.0-b'
+  VERSION = '0.1.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zuora_observability
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.b
+  version: 0.1.0
 platform: ruby
 authors:
 - Hartley McGuire
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-03 00:00:00.000000000 Z
+date: 2021-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: lograge
@@ -140,16 +140,16 @@ dependencies:
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop-rails
   requirement: !ruby/object:Gem::Requirement
@@ -168,16 +168,16 @@ dependencies:
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0.pre
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0.pre
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -261,11 +261,9 @@ files:
 - lib/zuora_observability/engine.rb
 - lib/zuora_observability/env.rb
 - lib/zuora_observability/logger.rb
+- lib/zuora_observability/logging/custom_options.rb
 - lib/zuora_observability/logging/formatter.rb
 - lib/zuora_observability/metrics.rb
-- lib/zuora_observability/metrics/page_request.rb
-- lib/zuora_observability/metrics/point_value.rb
-- lib/zuora_observability/metrics/telegraf.rb
 - lib/zuora_observability/version.rb
 homepage: 
 licenses:
@@ -279,12 +277,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.1.4
 signing_key: 

data/lib/zuora_observability/metrics/page_request.rb

@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module ZuoraObservability
-  module Metrics
-    # Object of this class is passed to the ActiveSupport::Notification hook
-    class PageRequest
-      # This method is triggered when a non error page is loaded (not 404)
-      def call(_name, started, finished, unique_id, payload)
-        # If the url contains any css or JavaScript files then do not collect metrics for them
-        return nil if %w[css assets jpg png jpeg ico].any? { |word| payload[:path].include?(word) }
-
-        # Getting the endpoint and the content_type
-        content_hash = { :html => 'text/html', :js => 'application/javascript', :json => 'application/json', :csv => 'text/csv' }
-        content_type = content_hash.key?(payload[:format]) ? content_hash[payload[:format]] : payload[:format]
-        content_type = content_type.to_s.gsub('text/javascript', 'application/javascript')
-
-        # payloads with 500 requests do not have status as it is not set by the controller
-        # https://github.com/rails/rails/issues/33335
-        # status_code = payload[:status] ? payload[:status] : payload[:exception_object].present? ? 500 : ""
-        if payload[:exception].present?
-          status_code, exception = [500, payload[:exception].first]
-        else
-          status_code, exception = [payload[:status], nil]
-        end
-
-        tags = { method: payload[:method], status: status_code, error_type: exception, content_type: content_type, controller: payload[:controller], action: payload[:action] }.compact
-
-        values = { view_time: payload[:view_runtime], db_time: payload[:db_runtime], response_time: ((finished - started) * 1000) }.compact
-        values = values.map { |k, v| [k, v.round(2)] }.to_h
-
-        ZuoraObservability::Metrics.write_to_telegraf(direction: :inbound, tags: tags, values: values)
-      end
-    end
-  end
-end

data/lib/zuora_observability/metrics/point_value.rb

@@ -1,84 +0,0 @@
-# this looks copied from https://github.com/influxdata/influxdb-ruby, it may be
-# worth just using the gem instead of vendoring the file
-# module InfluxDB
-module ZuoraObservability
-  module Metrics
-    # Convert data point to string using Line protocol
-    class PointValue
-      attr_reader :series, :values, :tags, :timestamp
-
-      def initialize(data)
-        @series    = escape data[:series], :measurement
-        @values    = escape_values data[:values]
-        @tags      = escape_tags data[:tags]
-        @timestamp = data[:timestamp]
-      end
-
-      def dump
-        dump =  @series.dup
-        dump << ",#{@tags}" if @tags
-        dump << " #{@values}"
-        dump << " #{@timestamp}" if @timestamp
-        dump
-      end
-
-      private
-
-      ESCAPES = {
-        measurement:  [' '.freeze, ','.freeze],
-        tag_key:      ['='.freeze, ' '.freeze, ','.freeze],
-        tag_value:    ['='.freeze, ' '.freeze, ','.freeze],
-        field_key:    ['='.freeze, ' '.freeze, ','.freeze, '"'.freeze],
-        field_value:  ["\\".freeze, '"'.freeze],
-      }.freeze
-
-      private_constant :ESCAPES
-
-      def escape(str, type)
-        # rubocop:disable Layout/AlignParameters
-        str = str.encode "UTF-8".freeze, "UTF-8".freeze,
-          invalid: :replace,
-          undef: :replace,
-          replace: "".freeze
-        # rubocop:enable Layout/AlignParameters
-
-        ESCAPES[type].each do |ch|
-          str = str.gsub(ch) { "\\#{ch}" }
-        end
-        str
-      end
-
-      def escape_values(values)
-        return if values.nil?
-        values.map do |k, v|
-          key = escape(k.to_s, :field_key)
-          val = escape_value(v)
-          "#{key}=#{val}"
-        end.join(",".freeze)
-      end
-
-      def escape_value(value)
-        if value.is_a?(String)
-          '"'.freeze + escape(value, :field_value) + '"'.freeze
-        elsif value.is_a?(Integer)
-          "#{value}i"
-        else
-          value.to_s
-        end
-      end
-
-      def escape_tags(tags)
-        return if tags.nil?
-
-        tags = tags.map do |k, v|
-          key = escape(k.to_s, :tag_key)
-          val = escape(v.to_s, :tag_value)
-
-          "#{key}=#{val}" unless key == "".freeze || val == "".freeze
-        end.compact
-
-        tags.join(",") unless tags.empty?
-      end
-    end
-  end
-end

data/lib/zuora_observability/metrics/telegraf.rb

@@ -1,98 +0,0 @@
-# frozen_string_literal: true
-
-module ZuoraObservability
-  module Metrics
-    # Functionality for sending metrics to a Telegraf endpoint
-    #
-    # it looks like https://github.com/influxdata/influxdb-ruby may provide some
-    # more high level abstractions instead of using a UDPSocket directly
-    class Telegraf
-      attr_accessor :host
-
-      OUTBOUND_METRICS = true
-      OUTBOUND_METRICS_NAME = 'request-outbound'
-      INBOUND_METRICS = true
-      INBOUND_METRICS_NAME = 'request-inbound'
-
-      def initialize
-        connect
-      end
-
-      def connect
-        # TODO(hartley): this Rails logger was originally ZuoraConnect.logger
-        Rails.logger.debug(format_metric_log('Telegraf', 'Need new connection')) if ZuoraObservability.configuration.telegraf_debug
-        uri = URI.parse(ZuoraObservability.configuration.telegraf_endpoint)
-        self.host = UDPSocket.new.tap do |socket|
-          socket.connect uri.host, uri.port
-        end
-      rescue => ex
-        self.host = nil
-        # TODO(hartley): this Rails logger was originally ZuoraConnect.logger
-        Rails.logger.warn(self.format_metric_log('Telegraf', "Failed to connect: #{ex.class}")) if Rails.env.to_s != 'production'
-      end
-
-      def write(direction: 'Unknown', tags: {}, values: {})
-        time = Benchmark.measure do
-          # To avoid writing metrics from rspec tests
-          if Rails.env.to_sym != :test
-            app_instance = Thread.current[:appinstance].present? ? Thread.current[:appinstance].id : 0
-            tags = {
-              app_name: Env.app_name, process_type: Env.process_type,
-              app_instance: app_instance, pod_name: Env.pod_name
-            }.merge(tags)
-
-            if direction == :inbound
-              # This condition relies on a monkey patch in the connect gem that
-              # adds a to_bool method for Nil, True, and False that are not
-              # present by default
-              if INBOUND_METRICS && !Thread.current[:inbound_metric].to_bool
-                self.write_udp(series: INBOUND_METRICS_NAME, tags: tags, values: values)
-                Thread.current[:inbound_metric] = true
-              else
-                return
-              end
-            elsif direction == :outbound
-              write_udp(series: OUTBOUND_METRICS_NAME, tags: tags, values: values) if OUTBOUND_METRICS
-            else
-              write_udp(series: direction, tags: tags, values: values)
-            end
-          end
-        end
-
-        return unless ZuoraObservability.configuration.telegraf_debug
-
-        # TODO(hartley): these Rails loggers were originally ZuoraConnect.logger
-        Rails.logger.debug(format_metric_log('Telegraf', tags.to_s))
-        Rails.logger.debug(format_metric_log('Telegraf', values.to_s))
-        Rails.logger.debug(
-          format_metric_log(
-            'Telegraf',
-            "Writing '#{direction.capitalize}': #{time.real.round(5)} ms"
-          )
-        )
-      end
-
-      def write_udp(series: '', tags: {}, values: {})
-        return if values.blank?
-
-        host.write PointValue.new({ series: series, tags: tags, values: values }).dump
-      rescue => ex
-        self.connect
-        ZuoraConnect.logger.warn(self.format_metric_log('Telegraf', "Failed to write udp: #{ex.class}")) if Rails.env.to_s != 'production'
-      end
-
-      def format_metric_log(message, dump = nil)
-        message_color = '1;91'
-        dump_color = '0;1'
-        log_entry = "  \e[#{message_color}m#{message}\e[0m   #{
-          "\e[#{dump_color}m#{dump}\e[0m" if dump
-        }"
-        if Rails.env.development?
-          log_entry
-        else
-          [message, dump].compact.join(' - ')
-        end
-      end
-    end
-  end
-end