contextualized_logs 0.0.1.pre.alpha

data/lib/contextualized_logs/contextualized_logger.rb

@@ -0,0 +1,90 @@
+require 'active_support'
+
+module ContextualizedLogs
+  # custom logger for
+  # logging in json format with log enrichment
+  # support Rails.logger.dump('msg', hash)
+
+  class ContextualizedLogger < ActiveSupport::Logger
+    def initialize(*args)
+      super(*args)
+      @formatter = formatter
+    end
+
+    def dump(msg, attributes, severity = :info)
+      # log message with attributes as structured dump attributes
+      send(severity, { msg: msg, attributes: attributes })
+    end
+
+    def dump_error(msg, attributes)
+      dump(msg, attributes, :error)
+    end
+
+    def current_context
+      CurrentContext.context
+    end
+
+    def formatter
+      Proc.new{|severity, timestamp, progname, msg|
+        # format (and enrich) log in JSON format (-> )
+        # https://docs.hq.com/logs/processing/attributes_naming_convention/#source-code
+        # correlation = Datadog.tracer.active_correlation
+        data = {
+          # dd: {
+          #   trace_id: correlation.trace_id,
+          #   span_id: correlation.span_id
+          # },
+          # ddsource: ['ruby'],
+          syslog: { env: Rails.env, host: Socket.gethostname },
+          type: severity.to_s,
+          time: timestamp
+        }
+        data[:stack] = Kernel.caller.
+          map { |caller| caller.gsub(/#{Rails.root}/, '') }.
+          reject { |caller| caller.start_with?('/usr/local') || caller.include?('/shared/bundle/') || caller.start_with?('/Users/') }.
+          first(15)
+        data[:log_type] = 'log'
+        data.merge!(parse_msg(msg)) # parse message (string, hash, error, ...)
+        data.merge!(current_context) # merge current request context
+        data.to_json + "\n"
+      }
+    end
+
+    private
+
+    def parse_msg(msg)
+      data = {}
+      case msg
+      when Hash
+        # used by logger.dump(msg|error, attributes = {})
+        if msg.include?(:attributes)
+          # adding message as log attributes if is a hash
+          data.merge!(parse_error(msg[:msg]))
+          data[:attributes] = msg[:attributes]
+        else
+          data.merge!(parse_error(msg))
+        end
+      else
+        data.merge!(parse_error(msg))
+      end
+      data
+    end
+
+    def parse_error(msg)
+      data = {}
+      case msg
+      when ::Exception
+        # format data to be interpreted as an error logs by
+        data[:error] = {
+          kind: msg.class.to_s,
+          message: msg.message,
+          stack: (msg.backtrace || []).join('; ')
+        }
+      else
+        data[:message] = msg.to_s
+      end
+      data
+    end
+  end
+
+end

data/lib/contextualized_logs/contextualized_model.rb

@@ -0,0 +1,54 @@
+require 'active_support'
+require_relative 'current_context'
+
+module ContextualizedLogs
+  module ContextualizedModel
+    extend ActiveSupport::Concern
+
+    DEFAULT_CURRENT_CONTEXT = ContextualizedLogs::CurrentContext
+
+    class_methods do
+      attr_reader :contextualizable_keys
+
+      def current_context
+        @current_context || DEFAULT_CURRENT_CONTEXT
+      end
+
+      private
+
+      def contextualizable(keys: {})
+        @contextualizable_keys = keys
+      end
+
+      def set_current_context(current_context)
+        @current_context = current_context
+      end
+    end
+
+    class << self
+      def contextualize(model, keys, context)
+        # Rails.logger.debug "model: #{model}"
+        # Rails.logger.debug "keys: #{keys}"
+        # Rails.logger.debug "context.context: #{context}"
+        # Rails.logger.debug "context.contextualized_model_enabled: #{context.contextualized_model_enabled}"
+        return unless context.contextualized_model_enabled
+        keys&.each do |k, v|
+          v = model.try(v.to_sym)
+          context.add_context(k, v) if v
+        end
+      end
+    end
+
+    included do
+      after_find do |object|
+        # Rails.logger.debug "after_find #{object}"
+        ContextualizedModel.contextualize(object, self.class.contextualizable_keys, self.class.current_context)
+      end
+
+      after_create do
+        # Rails.logger.debug "after_create #{self}"
+        ContextualizedModel.contextualize(self, self.class.contextualizable_keys, self.class.current_context)
+      end
+    end
+  end
+end

data/lib/contextualized_logs/contextualized_worker.rb

@@ -0,0 +1,41 @@
+require 'active_support'
+
+module ContextualizedLogs
+  module ContextualizedWorker
+    extend ActiveSupport::Concern
+
+    DEFAULT_CURRENT_CONTEXT = ContextualizedLogs::CurrentContext
+    DEFAULT_CONTEXTUALIZED_WORKER_ENABLED = false
+    DEFAULT_CONTEXTUALIZED_MODEL_ENABLED = false
+
+    class_methods do
+      # contextualize_args
+
+      def current_context
+        @current_context || DEFAULT_CURRENT_CONTEXT
+      end
+
+      def contextualized_worker_enabled
+        @contextualized_worker_enabled || DEFAULT_CONTEXTUALIZED_WORKER_ENABLED
+      end
+
+      def contextualized_model_enabled
+        @contextualized_model_enabled || DEFAULT_CONTEXTUALIZED_MODEL_ENABLED
+      end
+
+      private
+
+      def set_current_context(current_context)
+        @current_context = current_context
+      end
+
+      def contextualized_worker(enabled)
+        @contextualized_worker_enabled = enabled
+      end
+
+      def contextualized_model(enabled)
+        @contextualized_model_enabled = enabled
+      end
+    end
+  end
+end

data/lib/contextualized_logs/current_context.rb

@@ -0,0 +1,99 @@
+# https://github.com/rails/rails/pull/29180
+# storing global request info
+require 'active_support'
+
+module ContextualizedLogs
+  class CurrentContext < ActiveSupport::CurrentAttributes
+    # ⚠️ do not use this class to store any controller specific info..
+
+    attribute \
+      :request_uuid, :request_user_agent, :request_origin, :request_referer, :request_xhr, # request
+      :current_job_id, :enqueued_jobs_ids, :worker, :worker_args, # sidekiq
+      :request_remote_ip, :request_ip, :request_remote_addr, :request_x_forwarded_for, # ips
+      :errors,
+      :contextualized_model_enabled, # enable model context values
+      :context_values, :context_values_count, # context values
+      :resource_name # controller_action to correlate APM metrics
+
+    MAX_CONTEXT_VALUES = 100
+
+    def self.context
+      # https://docs.hq.com/logs/processing/attributes_naming_convention/#source-code
+
+      data = {}
+
+      data[:resource_name] = resource_name unless resource_name.nil?
+
+      #  normalized
+      data[:http] = {}
+      data[:http][:referer] = request_referer unless request_referer.nil?
+      data[:http][:request_id] = request_uuid unless request_uuid.nil?
+      data[:http][:useragent] = request_user_agent unless request_user_agent.nil?
+      data[:http][:origin] = request_origin unless request_origin.nil?
+      data.delete(:http) if data[:http].empty?
+
+      #  normalized
+      data[:network] = { client: {} }
+      data[:network][:client][:ip] = request_ip unless request_ip.nil?
+      data[:network][:client][:remote_addr] = request_remote_addr unless request_remote_addr.nil?
+      data[:network][:client][:remote_ip] = request_remote_ip unless request_remote_ip.nil?
+      data[:network][:client][:x_forwarded_for] = request_x_forwarded_for unless request_x_forwarded_for.nil?
+      data.delete(:network) if data[:network][:client].empty?
+
+      # eventual error response
+      #  normalized
+      data[:errors] = errors unless errors.nil?
+
+      # context_values
+      unless context_values.nil?
+        if context_values.is_a?(Hash) && !context_values.empty?
+          data[:context_values] = {}
+          context_values.each { |k, v| data[:context_values][k.to_sym] = v }
+        end
+      end
+
+      unless current_job_id.nil? && worker.nil?
+        data[:job] = { id: current_job_id, worker: worker }
+        data[:job][:args] = worker_args if worker_args
+      end
+
+      data
+    end
+
+    def self.to_json
+      attributes.to_json
+    end
+
+    def self.add_context(key, value)
+      self.context_values_count ||= 0
+      self.context_values_count += 1
+      if self.context_values_count >= MAX_CONTEXT_VALUES
+        Rails.logger.warn('high number of context values') if self.context_values_count == MAX_CONTEXT_VALUES
+        return
+
+      end
+      self.context_values ||= {}
+      self.context_values[key] ||= []
+      unless self.context_values[key].include?(value)
+        self.context_values[key] << value
+      end
+    end
+
+    def self.from_json(json)
+      return unless json
+
+      begin
+        values = JSON.parse(json).deep_symbolize_keys
+        values.each { |k, v| send("#{k}=", v) }
+      rescue
+      end
+    end
+
+    def self.add_error(error)
+      return if error.nil?
+
+      self.errors ||= []
+      self.errors << error
+    end
+  end
+end

data/lib/contextualized_logs/railtie.rb

@@ -0,0 +1,9 @@
+
+require "rails/railtie"
+require 'rails'
+
+module ContextualizedLogs
+  class Railtie < ::Rails::Railtie
+    railtie_name :contextualized_logs
+  end
+end

data/lib/contextualized_logs/sidekiq/middleware/client/inject_current_context.rb

@@ -0,0 +1,38 @@
+module ContextualizedLogs
+  # https://github.com/mperham/sidekiq/wiki/Middleware
+  module Sidekiq
+    module Middleware
+      module Client
+        class InjectCurrentContext
+          # @param [String, Class] worker_class the string or class of the worker class being enqueued
+          # @param [Hash] job the full job payload
+          #   * @see https://github.com/mperham/sidekiq/wiki/Job-Format
+          # @param [String] queue the name of the queue the job was pulled from
+          # @param [ConnectionPool] redis_pool the redis pool
+          # @return [Hash, FalseClass, nil] if false or nil is returned,
+          #   the job is not to be enqueued into redis, otherwise the block's
+          #   return value is returned
+          # @yield the next middleware in the chain or the enqueuing of the job
+          def call(worker_class, job, queue, redis_pool)
+            # https://github.com/rails/rails/issues/37526
+            # current attribute should be clear between jobs
+            # no need to `Current.reset`
+            worker_klass = worker_class.is_a?(String) ? worker_class.constantize : worker_class
+            if worker_klass.include?(ContextualizedWorker)
+              current_context = worker_klass.current_context
+              current_context.enqueued_jobs_ids ||= []
+              current_context.enqueued_jobs_ids << job['jid']
+              current_context.contextualized_model_enabled = worker_klass.contextualized_model_enabled
+              if worker_klass.contextualized_worker_enabled
+                job['context'] = current_context.to_json
+                Rails.logger.info "sidekiq: enqueing job #{worker_class}: #{job['jid']}, on queue: #{queue}"
+                Rails.logger.dump('Injecting context', JSON.parse(current_context.to_json), :debug)
+              end
+            end
+            yield
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contextualized_logs/sidekiq/middleware/server/restore_current_context.rb

@@ -0,0 +1,43 @@
+module ContextualizedLogs
+  # https://github.com/mperham/sidekiq/wiki/Middleware
+  module Sidekiq
+    module Middleware
+      module Server
+        # https://github.com/mperham/sidekiq/wiki/Middleware
+        class RestoreCurrentContext
+          # @param [Object] worker the worker instance
+          # @param [Hash] job the full job payload
+          #   * @see https://github.com/mperham/sidekiq/wiki/Job-Format
+          # @param [String] queue the name of the queue the job was pulled from
+          # @yield the next middleware in the chain or worker `perform` method
+          # @return [Void]
+          def call(worker, job, queue)
+            worker_klass = worker.class
+            if worker_klass.include?(ContextualizedWorker)
+              job_context_json = job['context']
+              current_context = worker_klass.current_context
+              current_context.from_json(job_context_json) if job_context_json
+              current_context.current_job_id = job['jid']
+              current_context.worker = worker.class.to_s
+              # https://github.com/mperham/sidekiq/wiki/Job-Format
+              current_context.worker_args = worker_klass.contextualize_args(job['args']) if worker_klass.respond_to?(:contextualize_args) && job['args']
+              current_context.contextualized_model_enabled = worker_klass.contextualized_model_enabled
+              if worker_klass.contextualized_worker_enabled
+                Rails.logger.info "sidekiq: performing job #{worker_klass}: #{job['jid']}, on queue #{queue}"
+                yield
+                Rails.logger.info "sidekiq: completing job #{worker_klass}: #{job['jid']}, on queue #{queue}"
+              end
+            else
+              yield
+            end
+          rescue StandardError => e
+            if worker_klass.include?(ContextualizedWorker)
+              Rails.logger.error "sidekiq: failure job #{worker.class}: #{job['jid']}, on queue #{queue}: #{e}"
+            end
+            raise e
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contextualized_logs/version.rb

@@ -0,0 +1,3 @@
+module ContextualizedLogs
+  VERSION = '0.0.1-alpha'
+end

data/lib/contextualized_logs.rb

@@ -0,0 +1,11 @@
+require "contextualized_logs/contextualized_logger"
+require "contextualized_logs/contextualized_controller"
+require "contextualized_logs/contextualized_model"
+require "contextualized_logs/current_context"
+require "contextualized_logs/sidekiq/middleware/client/inject_current_context"
+require "contextualized_logs/sidekiq/middleware/server/restore_current_context"
+require "contextualized_logs/contextualized_worker"
+
+module ContextualizedLogs
+  require "contextualized_logs/railtie" if defined?(Rails)
+end

data/lib/tasks/contextualized_logs_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :contextualized_logs do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: contextualized_logs
+version: !ruby/object:Gem::Version
+  version: 0.0.1.pre.alpha
+platform: ruby
+authors:
+- Hugues Bernet-Rollande
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-04-24 00:00:00.000000000 Z
+dependencies: []
+description: |
+  Online logging solution (like [Datadog](https://www.datadoghq.com)) have drastically transform the way we log.
+
+  An app will nowdays logs dozen (hundred) of logs per requests.
+
+  The issue is often to correlate this logs, with the initiating request (or job) and add shared metadata on this logs.
+
+  Here come `ContextualizedLogs`.
+
+  The main idea is to enhance your logs from your controller (including `ContextualizedController`, which use a before action), which will add the params to your logs (and some metadata about the request itself, like `request.uuid`).
+
+  This metadata are stored in a `ActiveSupport::CurrentAttributes` which is a singleton (reset per request).
+
+  Each subsequent logs in this thread (request) will also be enriched with this metadata, making it easier to find all the logs associated with a request (`uuid`, `ip`, `params.xxx`).
+
+  On top of this, logs can also be enriched by the ActiveRecord model they use (`create` or `find`) (models including `ContextualizedModel`). So any time a contextualized model is created or find, some metadata related to the model (`id`, ...) will also be added to the logs.
+
+  Allowing you to find all logs which "touched" this models.
+email:
+- hugues@xdev.fr
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/assets/config/manifest.js
+- app/assets/javascripts/application.js
+- app/assets/javascripts/cable.js
+- app/assets/stylesheets/application.css
+- app/channels/application_cable/channel.rb
+- app/channels/application_cable/connection.rb
+- app/controllers/application_controller.rb
+- app/controllers/model_controller.rb
+- app/helpers/application_helper.rb
+- app/jobs/application_job.rb
+- app/mailers/application_mailer.rb
+- app/models/application_record.rb
+- app/models/model.rb
+- app/views/layouts/application.html.erb
+- app/views/layouts/mailer.html.erb
+- app/views/layouts/mailer.text.erb
+- app/workers/model_worker.rb
+- config/application.rb
+- config/boot.rb
+- config/cable.yml
+- config/credentials.yml.enc
+- config/database.yml
+- config/environment.rb
+- config/environments/development.rb
+- config/environments/production.rb
+- config/environments/test.rb
+- config/initializers/application_controller_renderer.rb
+- config/initializers/assets.rb
+- config/initializers/backtrace_silencers.rb
+- config/initializers/content_security_policy.rb
+- config/initializers/cookies_serializer.rb
+- config/initializers/filter_parameter_logging.rb
+- config/initializers/inflections.rb
+- config/initializers/mime_types.rb
+- config/initializers/sidekiq.rb
+- config/initializers/wrap_parameters.rb
+- config/locales/en.yml
+- config/master.key
+- config/puma.rb
+- config/routes.rb
+- config/spring.rb
+- config/storage.yml
+- db/development.sqlite3
+- db/migrate/20200424081113_create_model.rb
+- db/schema.rb
+- db/seeds.rb
+- db/test.sqlite3
+- lib/contextualized_logs.rb
+- lib/contextualized_logs/contextualized_controller.rb
+- lib/contextualized_logs/contextualized_logger.rb
+- lib/contextualized_logs/contextualized_model.rb
+- lib/contextualized_logs/contextualized_worker.rb
+- lib/contextualized_logs/current_context.rb
+- lib/contextualized_logs/railtie.rb
+- lib/contextualized_logs/sidekiq/middleware/client/inject_current_context.rb
+- lib/contextualized_logs/sidekiq/middleware/server/restore_current_context.rb
+- lib/contextualized_logs/version.rb
+- lib/tasks/contextualized_logs_tasks.rake
+homepage: https://github.com/hugues/contextualized_logs
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: Contextualize your logs (requests params, found/created model metadata, workers,
+  ...)
+test_files: []