lumberjack_json_device 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '09791e96f24532aeb09e536ed838a28e26d08ed47a868f3f31698b1fefd12c29'
+  data.tar.gz: 4314fc238210e81ffb8743c660bf6cefe2921567e3611dd122b898542fe4578f
+SHA512:
+  metadata.gz: db055ceb22bfca8ebcd2b04454ef2bc4634a48af53d960e64196ff328f06eecee687bab2f8b776a203b3a7f52d3f131e5c2d4a8ff8525930ebfe06010d52ccef
+  data.tar.gz: bee629eeef6c71b6c19c39ec3d4e863baa796f053879202efb460aec88d6649f9127437bd4672a1dafb7a201e6de36f11eaf1e7cfb3f02768687216e77d808d0

data/CHANGE_LOG.md

@@ -0,0 +1,3 @@
+# 1.0.0
+
+* Initial release

data/MIT_LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2020 Brian Durand
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,110 @@
+# Lumberjack JSON Device
+
+[![Build Status](https://travis-ci.org/bdurand/lumberjack_json_device.svg?branch=master)](https://travis-ci.org/bdurand/lumberjack_json_device)
+[![Maintainability](https://api.codeclimate.com/v1/badges/c62cb886b86381560810/maintainability)](https://codeclimate.com/github/bdurand/lumberjack_json_device/maintainability)
+
+This gem provides a logging device for the [lumberjack](https://github.com/bdurand/lumberjack) gem that will log JSON formatted output to a stream with one JSON document per line. This can be used as part of a log processing pipeline to ship the log to a structured data store or logging service.
+
+## Destination
+
+You can send the JSON output either to a stream or to another Lumberjack device.
+
+```ruby
+# Send to STDOUT
+device = Lumberjack::JsonDevice.new(STDOUT)
+
+# Send to another logging device
+log_file = Lumberjack::Device::LogFile.new("/var/log/app.log")
+device = Lumberjack::JsonDevice.new(log_file)
+```
+
+## JSON Structure
+
+By default, the JSON document will map to the `Lumberjack::LogEntry` data structure.
+
+```json
+{"timestamp": "2020-01-02T19:47:45.123455", "severity": "INFO", "progname": "web", "pid": 101, "message": "test", "tags": {"foo": "bar"}}
+```
+
+You can specify a mapping to the device to customize the JSON document data structure. You can map the standard field names (time, severity, progname, pid, message, and tags) to custom field names.
+
+If you map a field to an array, it will be mapped into a nested hash in the JSON document.
+
+Any keys beyond the standard field names will be populated by a tag with the same name. These tags will not be included with the rest of the tags.
+
+```ruby
+device = Lumberjack::JsonDevice.new(STDOUT, mapping: {
+  time: "timestamp",
+  severity: "level",
+  progname: ["app", "name"],
+  pid: ["app", "pid"],
+  message: "message",
+  duration: "duration",
+  tags: "tags"
+})
+```
+
+```json
+{"timestamp": "2020-01-02T19:47:45.123455", "level": "INFO", "app": {"name": "web", "pid": 101}, "message": "test", "duration": 5, "tags": {"foo": "bar"}}
+```
+
+If you omit any fields in the mapping, they will not appear in the JSON document.
+
+```ruby
+device = Lumberjack::JsonDevice.new(STDOUT, mapping: {
+  time: "timestamp",
+  severity: "level",
+  message: "message",
+})
+```
+
+```json
+{"timestamp": "2020-01-02T19:47:45.123455", "level": "INFO", "message": "test"}
+```
+
+You can also provide a block or any object that responds to `call` in a mapping. The block will be called with the value and should emit a hash that will be merged into the JSON document.
+
+```ruby
+device = Lumberjack::JsonDevice.new(STDOUT, mapping: {
+  time: lambda { |val| (timestamp: val.to_f * 1000).round} },
+  severity: "level",
+  message: "test",
+})
+```
+
+```json
+{"timestamp": 1578125375588, "level": "INFO", "message": "test"}
+```
+
+Finally, you can specify `true` in the mapping as a short cut to map the field to the same name. If the field name contains periods, it will be mapped to a nested structure.
+
+```ruby
+device = Lumberjack::JsonDevice.new(STDOUT, mapping: {
+  "message" => true,
+  "http.status" => true,
+  "http.method" => true,
+  "http.path" => true
+})
+```
+
+```json
+{"message": "test", "http": {"status": 200, "method": "GET", "path": "/resource"}}
+```
+
+
+## Data Formatting
+
+The device will have a `Lumberjack::Formatter` that will be used to format objects before serializing them as JSON. You can add additional formatters for classes to the default formatter, or supply a custom one when creating the device.
+
+```ruby
+device.formatter.add(Exception, LumberjacK::Formatter::InspectFormatter.new)
+device.formatter.add(ActiveRecord::Base, LumberjacK::Formatter::IdFormatter.new)
+```
+
+You can also specify the `datetime_format` that will be used to serialize Time and DateTime objects.
+
+```ruby
+device.datetime_format = "%Y-%m-%dT%H:%M:%S.%3N"
+```
+
+Log entries with no message will not be written to the log.

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/lib/lumberjack_json_device.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require 'lumberjack'
+require 'multi_json'
+require 'thread'
+
+module Lumberjack
+  # This Lumberjack device logs output to another device as JSON formatted text with one document per line.
+  #
+  # The mapping parameter can be used to define the JSON data structure. To define the structure pass in a
+  # hash with key indicating the log entry field and the value indicating the JSON document key.
+  #
+  # The standard entry fields are mapped with the following keys:
+  #
+  # * :time
+  # * :severity
+  # * :progname
+  # * :pid
+  # * :message
+  # * :tags
+  #
+  # Any additional keys will be pulled from the tags. If any of the standard keys are missing or have a nil
+  # mapping, the entry field will not be included in the JSON output.
+  #
+  # You can create a nested JSON structure by specifying an array as the JSON key.
+  class JsonDevice < Device
+
+    DEFAULT_MAPPING = {
+      time: true,
+      severity: true,
+      progname: true,
+      pid: true,
+      message: true,
+      tags: true
+    }.freeze
+
+    DEFAULT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%6N%z"
+
+    attr_accessor :formatter
+    attr_reader :mapping
+
+    def initialize(stream_or_device, mapping: DEFAULT_MAPPING, formatter: nil, datetime_format: nil)
+      @mutex = Mutex.new
+
+      if stream_or_device.is_a?(Device)
+        @device = stream_or_device
+      else
+        @device = Writer.new(stream_or_device)
+      end
+
+      self.mapping = mapping
+
+      if formatter
+        @formatter = formatter
+      else
+        @formatter = default_formatter
+        datetime_format = DEFAULT_TIME_FORMAT if datetime_format.nil?
+      end
+      add_datetime_formatter!(datetime_format) unless datetime_format.nil?
+    end
+
+    def write(entry)
+      return if entry.message.nil? || entry.message == ""
+      data = entry_as_json(entry)
+      json = MultiJson.dump(data)
+      @device.write(json)
+    end
+
+    def flush
+      @device.flush
+    end
+
+    def datetime_format
+      @datetime_format
+    end
+
+    # Set the datetime format for the log timestamp.
+    def datetime_format=(format)
+      add_datetime_formatter!(format)
+    end
+
+    # Set the mapping for how to map an entry to a JSON object.
+    def mapping=(mapping)
+      @mutex.synchronize do
+        keys = {}
+        mapping.each do |key, value|
+          if value == true
+            value = key.to_s.split(".")
+            value = value.first if value.size == 1
+          end
+          keys[key.to_sym] = value if value
+        end
+
+        @time_key = keys.delete(:time)
+        @severity_key = keys.delete(:severity)
+        @progname_key = keys.delete(:progname)
+        @pid_key = keys.delete(:pid)
+        @message_key = keys.delete(:message)
+        @tags_key = keys.delete(:tags)
+        @custom_keys = keys
+        @mapping = mapping
+      end
+      nil
+    end
+
+    def map(field_mapping)
+      new_mapping = {}
+      field_mapping.each do |key, value|
+        new_mapping[key.to_sym] = value
+      end
+      self.mapping = mapping.merge(new_mapping)
+    end
+
+    # Convert a Lumberjack::LogEntry to a Hash using the specified field mapping.
+    def entry_as_json(entry)
+      data = {}
+      set_attribute(data, @time_key, entry.time) unless @time_key.nil?
+      set_attribute(data, @severity_key, entry.severity_label) unless @severity_key.nil?
+      set_attribute(data, @progname_key, entry.progname) unless @progname_key.nil?
+      set_attribute(data, @pid_key, entry.pid) unless @pid_key.nil?
+      set_attribute(data, @message_key, entry.message) unless @message_key.nil?
+
+      tags = entry.tags
+      if @custom_keys.size > 0
+        tags = (tags.nil? ? {} : tags.dup)
+        @custom_keys.each do |name, key|
+          set_attribute(data, key, tags.delete(name.to_s))
+        end
+      end
+
+      unless @tags_key.nil?
+        tags ||= {}
+        set_attribute(data, @tags_key, tags)
+      end
+
+      data = @formatter.format(data) if @formatter
+      data
+    end
+
+    private
+
+    def set_attribute(data, key, value)
+      return if value.nil?
+
+      if (value.is_a?(Time) || value.is_a?(DateTime)) && @time_formatter
+        value = @time_formatter.call(value)
+      end
+
+      if key.is_a?(Array)
+        unless key.empty?
+          if key.size == 1
+            data[key.first] = value
+          else
+            data[key.first] ||= {}
+            set_attribute(data[key.first], key[1, key.size], value)
+          end
+        end
+      elsif key.respond_to?(:call)
+        hash = key.call(value)
+        if hash.is_a?(Hash)
+          deep_merge!(data, Lumberjack::Tags.stringify_keys(hash))
+        end
+      else
+        data[key] = value unless key.nil?
+      end
+    end
+
+    def default_formatter
+      formatter = Formatter.new.clear
+      object_formatter = Lumberjack::Formatter::ObjectFormatter.new
+      formatter.add(String, object_formatter)
+      formatter.add(Object, object_formatter)
+      formatter.add(Enumerable, Formatter::StructuredFormatter.new(formatter))
+      formatter
+    end
+
+    def add_datetime_formatter!(datetime_format)
+      if datetime_format
+        @datetime_format = datetime_format
+        time_formatter = Lumberjack::Formatter::DateTimeFormatter.new(datetime_format)
+        formatter.add(Time, time_formatter)
+        formatter.add(Date, time_formatter)
+      else
+        @datetime_format = nil
+        formatter.remove(Time)
+        formatter.remove(Date)
+      end
+    end
+
+    def deep_merge!(hash, other_hash, &block)
+      hash.merge!(other_hash) do |key, this_val, other_val|
+        if this_val.is_a?(Hash) && other_val.is_a?(Hash)
+          deep_merge!(this_val, other_val, &block)
+        elsif block_given?
+          block.call(key, this_val, other_val)
+        else
+          other_val
+        end
+      end
+    end
+
+  end
+end

data/lumberjack_json_device.gemspec

@@ -0,0 +1,34 @@
+Gem::Specification.new do |spec|
+  spec.name = 'lumberjack_json_device'
+  spec.version = File.read(File.expand_path("../VERSION", __FILE__)).strip
+  spec.authors = ['Brian Durand']
+  spec.email = ['bbdurand@gmail.com']
+
+  spec.summary = "A logging device for the lumberjack gem that writes log entries as JSON documentspec."
+  spec.homepage = "https://github.com/bdurand/lumberjack_json_device"
+  spec.license = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  ignore_files = %w(
+    .gitignore
+    .travis.yml
+    Appraisals
+    Gemfile
+    Gemfile.lock
+    Rakefile
+    gemfiles/
+    spec/
+  )
+  spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject{ |f| ignore_files.any?{ |path| f.start_with?(path) } }
+  end
+
+  spec.require_paths = ['lib']
+
+  spec.add_dependency "lumberjack", ">=1.2"
+  spec.add_dependency "multi_json"
+
+  spec.add_development_dependency("rspec", ["~> 3.0"])
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: lumberjack_json_device
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Brian Durand
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-01-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: lumberjack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- bbdurand@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGE_LOG.md
+- MIT_LICENSE
+- README.md
+- VERSION
+- lib/lumberjack_json_device.rb
+- lumberjack_json_device.gemspec
+homepage: https://github.com/bdurand/lumberjack_json_device
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: A logging device for the lumberjack gem that writes log entries as JSON documentspec.
+test_files: []