lumberjack_ecs_device 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: edb8e82b2c9f2ddd629c9efddf43f63ac4f5b984b754a9466c5d714e22727f0b
+  data.tar.gz: 52bd32275d8e55f1e03cef9aa661e19fcf4de2fe5296c277dcf161cb523a1258
+SHA512:
+  metadata.gz: 2f0a44ee035ae0053a1d18927051f57d470dbb12b913a18c9e363897ad1148cd74f3d7dd62929377438e9d0657596338ef11b0f32d0951959c8b72e8c7346a29
+  data.tar.gz: 3ce536acc0ba277dc0d193bc5fa1dd8fb0bf779cc53f9235b6f749d9842a11388d8b5eb03f8c53b7125d1d5f4400f6ea3fa5d3919e606800d64eda6b144129a7

data/CHANGE_LOG.md

@@ -0,0 +1,3 @@
+# 1.0.0
+
+* Initial release

data/MIT_LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2021 Brian Durand
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+# Lumberjack ECS Device
+
+![Continuous Integration](https://github.com/bdurand/lumberjack_ecs_device/workflows/Continuous%20Integration/badge.svg)
+[![Maintainability](https://api.codeclimate.com/v1/badges/97e98dc4d3d2565a3208/maintainability)](https://codeclimate.com/github/bdurand/lumberjack_ecs_device/maintainability)
+[![Ruby Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://github.com/testdouble/standard)
+
+This gem provides a logging device that produces JSON output that matches the standard fields defined for the [Elastic Common Schema](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html). This allows logs to be sent seamlessly to Kibana or other servers that expect this format.
+
+* The time will be sent as "@timestamp" with a precision in microseconds.
+
+* The severity will be sent as "log.level" with a string label (DEBUG, INFO, WARN, ERROR, FATAL).
+
+* The progname will be sent as "process.name"
+
+* The pid will be sent as "process.pid".
+
+* The message will be sent as "message". In addition, if the message is an exception, the error message, class, and backtrace will be sent as "error.message", "error.type", and "error.stack_trace".
+
+* If the "error" tag contains an exception, it will be sent as "error.message", "error.type", and "error.stack_trace".
+
+* A duration can be sent as a number of seconds in the "duration" tag or as a number of milliseconds in the "duration_ms" tag or as a number of microsectons in the "duration_micros" tag or as a number of nanoseconds in the "duration_ns" tag. The value will be sent as "event.duration" and converted to nanoseconds.
+
+* All other log tags are sent as is. If a tag name includes a dot, it will be sent as a nested JSON structure.
+
+This device extends from [`Lumberjack::JsonDevice`](). It is not tied to ECS or Kibana in any way other than that it is opinionated about how to map and format some log tags. It can be used with other services or pipelines without issue.
+
+## Example
+
+You could log an HTTP request to some of the ECS standard fields like this:
+
+```ruby
+logger.tag("http.request.method" => request.method, "url.full" => request.url) do
+  logger.info("#{request.method} #{request.path} finished in #{elapsed_time} seconds",
+    duration: elapsed_time,
+    "http.response.status_code" => response.status
+  )
+end
+```

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/lib/lumberjack_ecs_device.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+require "lumberjack_json_device"
+
+module Lumberjack
+  # This Lumberjack device logs output to another device as JSON formatted text that maps fields
+  # to the standard ECS JSON format.
+  #
+  # See https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html
+  class EcsDevice < JsonDevice
+    module ExceptionHash
+      protected
+
+      def exception_hash(exception, device)
+        hash = {"type" => exception.class.name}
+        hash["message"] = exception.message unless exception.message.nil?
+        trace = exception.backtrace
+        if trace && device && device.respond_to?(:backtrace_cleaner) && device.backtrace_cleaner
+          trace = device.backtrace_cleaner.call(trace)
+        end
+        hash["stack_trace"] = trace if trace
+        hash
+      end
+    end
+
+    # Formatter to format a messge as an error if it is an exception.
+    class MessageExceptionFormatter
+      include ExceptionHash
+
+      def initialize(device = nil)
+        @device = device
+      end
+
+      def call(object)
+        if object.is_a?(Exception)
+          {
+            "message" => object.inspect,
+            "error" => exception_hash(object, @device)
+          }
+        elsif object.is_a?(Hash)
+          {"message" => object}
+        elsif object.nil?
+          {"message" => nil}
+        else
+          message = object.to_s
+          max_message_length = @device.max_message_length
+          if max_message_length && message.length > max_message_length
+            message = message[0, max_message_length]
+          end
+          {"message" => message}
+        end
+      end
+    end
+
+    # Formatter to remove empty tag values and expand the error tag if it is an exception.
+    class EcsTagsFormatter
+      include ExceptionHash
+
+      def initialize(device = nil)
+        @device = device
+      end
+
+      def call(tags)
+        copy = {}
+        tags.each do |name, value|
+          value = remove_empty_values(value)
+          next if value.nil?
+
+          if name == "error" && value.is_a?(Exception)
+            copy[name] = exception_hash(value, @device)
+          elsif name.include?(".")
+            names = name.split(".")
+            next_value_in_hash(copy, names, value)
+          else
+            copy[name] = value
+          end
+        end
+        copy
+      end
+
+      private
+
+      def remove_empty_values(value)
+        if value.is_a?(String)
+          value unless value.empty?
+        elsif value.is_a?(Hash)
+          new_hash = {}
+          value.each do |key, val|
+            val = remove_empty_values(val)
+            new_hash[key] = val unless val.nil?
+          end
+          new_hash unless new_hash.empty?
+        elsif value.is_a?(Array)
+          new_array = value.collect { |val| remove_empty_values(val) }
+          new_array unless new_array.empty?
+        else
+          value
+        end
+      end
+
+      def next_value_in_hash(hash, keys, value)
+        key = keys.first
+        if keys.size == 1
+          hash[key] = value
+        else
+          current = hash[key]
+          unless current.is_a?(Hash)
+            current = {}
+            hash[key] = current
+          end
+          next_value_in_hash(current, keys[1, keys.size], value)
+        end
+      end
+    end
+
+    class DurationFormatter
+      def initialize(multiplier)
+        @multiplier = multiplier
+      end
+
+      def call(value)
+        if value.is_a?(Numeric)
+          value = (value.to_f * @multiplier).round
+        end
+        {"event" => {"duration" => value}}
+      end
+    end
+
+    ECS_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S.%6NZ"
+
+    # You can specify a backtrace cleaner that will be called with exception backtraces before they
+    # are added to the payload. You can use this to remove superfluous lines, compress line length, etc.
+    # One use for it is to keep stack traces clean and prevent them from overflowing the limit on
+    # the payload size for an individual log entry.
+    attr_accessor :backtrace_cleaner
+
+    # You can specify a limit on the message size. Messages over this size will be split into multiple
+    # log entries to prevent overflowing the limit on message size which makes the log entries unparseable.
+    attr_accessor :max_message_length
+
+    def initialize(stream_or_device, backtrace_cleaner: nil, max_message_length: nil, datetime_format: ECS_TIMESTAMP_FORMAT)
+      super(stream_or_device, mapping: ecs_mapping, datetime_format: datetime_format)
+      self.backtrace_cleaner = backtrace_cleaner
+      self.max_message_length = max_message_length
+      @utc_timestamps = !!datetime_format.match(/[^%]Z/)
+    end
+
+    def entry_as_json(entry)
+      original_time = entry.time
+      begin
+        entry.time = entry.time.utc if @utc_timestamps && !entry.time.utc?
+        super
+      ensure
+        entry.time = original_time
+      end
+    end
+
+    private
+
+    def ecs_mapping
+      {
+        time: "@timestamp",
+        severity: ["log", "level"],
+        progname: ["process", "name"],
+        pid: ["process", "pid"],
+        message: MessageExceptionFormatter.new(self),
+        duration: DurationFormatter.new(1_000_000_000),
+        duration_ms: DurationFormatter.new(1_000_000),
+        duration_micros: DurationFormatter.new(1_000),
+        duration_ns: ["event", "duration"],
+        tags: EcsTagsFormatter.new(self)
+      }
+    end
+  end
+end

data/lumberjack_ecs_device.gemspec

@@ -0,0 +1,28 @@
+Gem::Specification.new do |spec|
+  spec.name = "lumberjack_ecs_device"
+  spec.version = File.read(File.expand_path("../VERSION", __FILE__)).strip
+  spec.authors = ["Brian Durand"]
+  spec.email = ["bbdurand@gmail.com"]
+
+  spec.summary = "A logging device for formatting logs in Elastic Container Schema (ECS) format for integration with Kibana."
+  spec.homepage = "https://github.com/bdurand/lumberjack_ecs_device"
+  spec.license = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  ignore_files = %w[
+    .
+    Gemfile
+    Gemfile.lock
+    Rakefile
+    gemfiles/
+    spec/
+  ]
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject { |f| ignore_files.any? { |path| f.start_with?(path) } }
+  end
+
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "lumberjack_json_device", ">=1.0"
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: lumberjack_ecs_device
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Brian Durand
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-01-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: lumberjack_json_device
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: 
+email:
+- bbdurand@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGE_LOG.md
+- MIT_LICENSE
+- README.md
+- VERSION
+- lib/lumberjack_ecs_device.rb
+- lumberjack_ecs_device.gemspec
+homepage: https://github.com/bdurand/lumberjack_ecs_device
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: A logging device for formatting logs in Elastic Container Schema (ECS) format
+  for integration with Kibana.
+test_files: []