unified_logger 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2b1182b4653c380d8b35b55733c0ba47fa4c5eff722178765dfe6b2f1fddaf8c
+  data.tar.gz: 7b94cd76f7288dffa0004f69a57e633ffa2342cfb67969031e8822f02a96aab6
+SHA512:
+  metadata.gz: 3a6090da5a5deb10ad56cf57c475342517dc854658bd23934b4a4c7bf247060cb0851a96af50698b8dd1ca268c8f2a147a4e7478df05931fd3cf0c7e2a3093f6
+  data.tar.gz: 23df148c3859f4e42f1646ddc94d774de39ed9908e74660035ebbe618f8b81d709d9fc5dbe96a0454d71ae4058c2df0ec2a65799d6d3e68caee49883d5d88b8e

data/lib/unified_logger/job_logger.rb

@@ -0,0 +1,45 @@
+require "English"
+
+module UnifiedLogger
+  class JobLogger
+    DEFAULT_MAX_RETRIES = 5
+
+    class << self
+      def log(job)
+        yield
+      ensure
+        log_execution(job) if UnifiedLogger.current_logger.is_a?(UnifiedLogger::Logger)
+      end
+
+      private
+
+      def log_execution(job)
+        log = {
+          log_type:             :job,
+          class_name:           job.class.name,
+          id:                   job.job_id,
+          queue:                job.queue_name,
+          params:               job.arguments,
+          executions_count:     job.executions,
+          exception_executions: job.exception_executions,
+          enqueued_at:          job.enqueued_at,
+          locale:               job.locale,
+          duration:             job.enqueued_at.present? ? UnifiedLogger.current_time - job.enqueued_at.in_time_zone : "unknown"
+        }
+        log[:custom] = UnifiedLogger::Logger.fetch_and_reset_custom_logs if UnifiedLogger::Logger.custom_logs.any?
+
+        if $ERROR_INFO
+          log[:exception] = UnifiedLogger::Logger.format_exception($ERROR_INFO)
+          log[:status] = job.executions >= DEFAULT_MAX_RETRIES ? :error : :warn
+        else
+          log[:status] = :ok
+        end
+
+        custom = {}
+        UnifiedLogger.transform_job_log_callable&.call(custom)
+        log.merge!(custom)
+        UnifiedLogger.current_logger.write(UnifiedLogger::Logger.format(log))
+      end
+    end
+  end
+end

data/lib/unified_logger/logger.rb

@@ -0,0 +1,143 @@
+module UnifiedLogger
+  class Logger < ::Logger
+    CUSTOM_LOGS = Concurrent::ThreadLocalVar.new([])
+    SEVERITY_LEVELS = {
+      debug:   ::Logger::DEBUG,
+      info:    ::Logger::INFO,
+      warn:    ::Logger::WARN,
+      error:   ::Logger::ERROR,
+      fatal:   ::Logger::FATAL,
+      unknown: ::Logger::UNKNOWN
+    }.freeze
+    SEVERITY_MAP = SEVERITY_LEVELS.invert.freeze
+
+    def initialize(logging_device = $stdout, *)
+      super
+      @logging_device = logging_device
+      self.formatter = proc {}
+    end
+
+    def debug(message = nil, params = {})
+      add(::Logger::DEBUG, message, params)
+    end
+
+    def info(message = nil, params = {})
+      add(::Logger::INFO, message, params)
+    end
+
+    def warn(message = nil, params = {})
+      add(::Logger::WARN, message, params)
+    end
+
+    def error(message = nil, params = {})
+      add(::Logger::ERROR, message, params)
+    end
+
+    def fatal(message = nil, params = {})
+      add(::Logger::FATAL, message, params)
+    end
+
+    def unknown(message = nil, params = {})
+      add(::Logger::UNKNOWN, message, params)
+    end
+
+    def add(severity, message = nil, params = {})
+      return true if message.blank?
+      return true unless severity >= level
+
+      severity_symbol = SEVERITY_MAP[severity] || :unknown
+      self.class.append_custom_log(severity_symbol, message, params)
+    end
+
+    def <<(message)
+      add(::Logger::UNKNOWN, message.to_s.chomp, {})
+      self
+    end
+
+    def write(message)
+      @logging_device.write("#{message}\n") if @logging_device.respond_to?(:write)
+    end
+
+    class << self
+      def custom_logs
+        CUSTOM_LOGS.value
+      end
+
+      def reset_thread_logs
+        CUSTOM_LOGS.value = []
+      end
+
+      def fetch_and_reset_custom_logs
+        logs = custom_logs
+        reset_thread_logs
+        logs
+      end
+
+      def trim(data)
+        data = filter(data)
+        size = data.inspect.length
+        max = UnifiedLogger.config[:max_log_field_size]
+        return data if size < max
+
+        begin
+          json = JSON.generate(data, quirks_mode: true)
+        rescue JSON::GeneratorError, Encoding::UndefinedConversionError
+          json = data.respond_to?("to_s") ? data.to_s : "unparseable object (instance of #{data.class.name})"
+        end
+        "#{json[0..max]}... (#{size - max} extra characters omitted)"
+      end
+
+      def format_exception(exception)
+        if exception.is_a?(String)
+          { message: exception }
+        elsif exception.is_a?(Exception)
+          btc = ActiveSupport::BacktraceCleaner.new
+          prefix = UnifiedLogger.backtrace_root + File::SEPARATOR
+          btc.add_filter { |line| line.sub(/\A#{Regexp.escape(prefix)}/, "") }
+          btc.add_silencer { |line| /request_logger.rb/.match?(line) }
+          { class_name: exception.class.name, message: exception.message, backtrace: btc.clean(exception.backtrace || []) }
+        elsif exception.respond_to?(:to_s)
+          exception.to_s
+        else
+          exception.class.name
+        end
+      end
+
+      def filter(content)
+        return content unless content.respond_to?(:each)
+
+        filter_class = if defined?(ActiveSupport::ParameterFilter)
+                         ActiveSupport::ParameterFilter
+                       else
+                         ActionDispatch::Http::ParameterFilter
+                       end
+        filter_class.new(UnifiedLogger.config[:filter_params]).filter(content)
+      end
+
+      def format(log)
+        filtered_log = filter(log)
+        formatter = UnifiedLogger.log_transformer
+        formatter.present? ? formatter.call(filtered_log) : filtered_log.to_json
+      end
+
+      def append_custom_log(severity, message, params)
+        clean_message = clean_log_message(message)
+        log_hash = { timestamp: UnifiedLogger.current_time, severity: severity,
+                     message: clean_message, params: params }.compact_blank
+
+        CUSTOM_LOGS.value = CUSTOM_LOGS.value + [log_hash]
+        true
+      end
+
+      def clean_log_message(text)
+        return text unless text.is_a?(String)
+
+        text = text.gsub(/\e\[[0-9;]*m/, "")
+        text = text.gsub(%r{[^a-zA-Z0-9\s.,;:!?'"()\[\]{}\-_@#$%&*+=<>|~`/]}, "")
+        text = text.gsub('"', "'")
+        text = text.gsub(/\s+/, " ")
+        text.strip
+      end
+    end
+  end
+end

data/lib/unified_logger/railtie.rb

@@ -0,0 +1,7 @@
+module UnifiedLogger
+  class Railtie < Rails::Railtie
+    initializer "unified_logger.middleware", after: :load_config_initializers do |app|
+      app.middleware.insert_after ActionDispatch::DebugExceptions, UnifiedLogger::RequestLogger if UnifiedLogger.config[:auto_insert_middleware]
+    end
+  end
+end

data/lib/unified_logger/request_logger.rb

@@ -0,0 +1,108 @@
+require "English"
+
+module UnifiedLogger
+  class RequestLogger
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      return @app.call(env) unless UnifiedLogger.current_logger.is_a?(UnifiedLogger::Logger)
+
+      started = UnifiedLogger.current_time
+      status, headers, response = @app.call(env)
+    ensure
+      if UnifiedLogger.current_logger.is_a?(UnifiedLogger::Logger) && !silenced?(env["REQUEST_PATH"])
+        log = build_log(started, env, status, headers, response)
+        custom = {}
+        UnifiedLogger.transform_request_log_callable&.call(custom, env)
+        log.merge!(custom)
+        UnifiedLogger.current_logger.write(UnifiedLogger::Logger.format(log))
+      end
+    end
+
+    private
+
+    def silenced?(path)
+      UnifiedLogger.config[:silence_paths].any? do |pattern|
+        pattern.is_a?(Regexp) ? pattern.match?(path) : pattern == path
+      end
+    end
+
+    def build_log(started, env, status = nil, headers = nil, response = nil)
+      status   ||= 500
+      headers  ||= {}
+      response ||= ""
+      path_parameters = env["action_dispatch.request.path_parameters"] || {}
+      query_string = env["QUERY_STRING"]
+      log = {
+        log_type:   :request,
+        id:         env["action_dispatch.request_id"],
+        ip:         env["action_dispatch.remote_ip"].to_s,
+        controller: path_parameters[:controller],
+        action:     path_parameters[:action],
+        request:    {
+          path:         env["REQUEST_PATH"],
+          method:       env["REQUEST_METHOD"],
+          headers:      env.select { |e| e.start_with?("HTTP_") }.reject { |h| h.start_with?("HTTP_SEC_") },
+          path_params:  path_parameters.except(:controller, :action),
+          query_params: query_string.blank? ? {} : Rack::Utils.parse_nested_query(query_string),
+          body:         request_body(env)
+        },
+        response:   {
+          headers: headers,
+          status:  status,
+          body:    response_body(response, headers["content-type"])
+        },
+        thread_id:  Thread.current.object_id,
+        process_id: Process.pid,
+        duration:   started ? UnifiedLogger.current_time - started : 0
+      }
+      log[:exception] = UnifiedLogger::Logger.format_exception($ERROR_INFO) if $ERROR_INFO.present?
+      log[:custom] = UnifiedLogger::Logger.fetch_and_reset_custom_logs if UnifiedLogger::Logger.custom_logs.any?
+
+      log
+    end
+
+    def request_body(env)
+      return nil unless (input = env["rack.input"])
+      return env["rack.request.form_hash"] if env["rack.request.form_hash"]
+
+      input.rewind
+      body = input.read
+      input.rewind
+
+      parse_body(body, env["CONTENT_TYPE"])
+    end
+
+    def response_body(response, content_type)
+      return nil if response.nil? || (response.respond_to?(:empty?) && response.empty?)
+      return nil if content_type&.exclude?("application/") && content_type.exclude?("text/plain")
+      return response_body(response.body, content_type) if response.respond_to?(:body)
+
+      body = response.respond_to?(:map) ? response.map(&:to_s).join : response.to_s
+
+      parse_body(body, content_type)
+    ensure
+      response.close if response.respond_to?(:close)
+    end
+
+    def parse_body(body, content_type)
+      return nil if body.nil?
+      return nil if body.empty?
+
+      main_content_type = content_type&.split(";")&.first&.strip&.downcase
+
+      case main_content_type
+      when "application/json"
+        JSON.parse(body)
+      when "application/x-www-form-urlencoded"
+        Rack::Utils.parse_nested_query(body)
+      else
+        UnifiedLogger::Logger.trim(body)
+      end
+    rescue JSON::ParserError, TypeError, ArgumentError
+      body
+    end
+  end
+end

data/lib/unified_logger/version.rb

@@ -0,0 +1,3 @@
+module UnifiedLogger
+  VERSION = "0.1.0".freeze
+end

data/lib/unified_logger.rb

@@ -0,0 +1,81 @@
+require "concurrent"
+require "active_support/core_ext/object/blank"
+require "active_support/core_ext/enumerable"
+require "active_support/parameter_filter"
+require "active_support/backtrace_cleaner"
+require "json"
+require "logger"
+
+require_relative "unified_logger/version"
+require_relative "unified_logger/logger"
+require_relative "unified_logger/request_logger"
+require_relative "unified_logger/job_logger"
+
+module UnifiedLogger
+  class DoubleDefineError < StandardError; end
+
+  DEFAULTS = {
+    max_log_field_size:     2048,
+    filter_params:          %i[passw secret token crypt salt certificate otp ssn set-cookie http_authorization http_cookie pin],
+    auto_insert_middleware: true,
+    silence_paths:          []
+  }.freeze
+
+  def self.config
+    @config ||= DEFAULTS.dup
+  end
+
+  def self.configure(options = {})
+    config.merge!(options)
+  end
+
+  class << self
+    attr_reader :transform_request_log_callable, :transform_job_log_callable, :log_transformer
+
+    def transform_request_log=(callable)
+      raise DoubleDefineError, "transform_request_log already defined" if @transform_request_log_callable
+
+      @transform_request_log_callable = callable
+    end
+
+    def transform_job_log=(callable)
+      raise DoubleDefineError, "transform_job_log already defined" if @transform_job_log_callable
+
+      @transform_job_log_callable = callable
+    end
+
+    def log_transformer=(callable)
+      raise DoubleDefineError, "log_transformer already defined" if @log_transformer
+
+      @log_transformer = callable
+    end
+
+    delegate :trim, :filter, :format, :format_exception,
+             :clean_log_message, :append_custom_log,
+             :custom_logs, :fetch_and_reset_custom_logs, :reset_thread_logs,
+             to: :"UnifiedLogger::Logger"
+  end
+
+  def self.backtrace_root
+    if defined?(Rails)
+      Rails.root.to_s
+    else
+      Dir.pwd
+    end
+  end
+
+  def self.current_logger
+    return unless defined?(Rails)
+
+    return Rails.logger if Rails.logger.is_a?(UnifiedLogger::Logger)
+    return Rails.logger unless Rails.logger.respond_to?(:broadcasts)
+
+    Rails.logger.broadcasts.find { |l| l.is_a?(UnifiedLogger::Logger) } || Rails.logger
+  end
+
+  def self.current_time
+    Time.zone&.now || Time.now.utc
+  end
+end
+
+require "unified_logger/railtie" if defined?(Rails::Railtie)

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: unified_logger
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Marcovecchio
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+description: One JSON log line per request or job. Captures request/response data,
+  in-app logger calls, exceptions with cleaned backtraces, and sensitive data filtering
+  — all in a single event.
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/unified_logger.rb
+- lib/unified_logger/job_logger.rb
+- lib/unified_logger/logger.rb
+- lib/unified_logger/railtie.rb
+- lib/unified_logger/request_logger.rb
+- lib/unified_logger/version.rb
+homepage: https://github.com/marcovecchio/unified_logger
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+  source_code_uri: https://github.com/marcovecchio/unified_logger
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: Structured JSON logging for Rack and Rails applications
+test_files: []