lorekeeper 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 71ffd469643f224ae00f00a43a2d1b3722c1bf6b
+  data.tar.gz: 77ed9aee28a33c8066fb3a2a9c73f07ad71ecbf3
+SHA512:
+  metadata.gz: 84a7111da82f6cb7d23ae21c27bdf72c6af712a270f98dda7f771d3ba3934c15d9df47f1cbdbfe40655ec48c149c52f4a10c76e8ade212a8eaf21ee8fd40b323
+  data.tar.gz: 7de9032c4d6aba109fa72dc9f50b73e40256ff668cc6cd79b5a1120c4826bd1abffd2f155060361cdeabbb43ab09aace4b39a123d1a74fa01c7a36f98ed6fbd3

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.byebug_history

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 2.1.1
+  - 2.2.2
+  - 2.3.0
+bundler_args: --without development

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+group :development do
+  gem "mutant-rspec", '~> 0.8'
+end
+
+# Specify your gem's dependencies in lorekeeper.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Jordi Polo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,104 @@
+# Lorekeeper
+
+LoreKeeper contains a highly optimized JSON logger. It outputs messages as JSON and let the users to add their own customized fields.
+When used without extra fields it outputs 20% faster than the standard Logger for messages not longer than one line of text.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'lorekeeper'
+```
+
+And then execute:
+
+    $ bundle
+
+
+## Usage
+
+### Normal logging methods
+
+LoreKeeper::JSONLogger API is compatible with standard lib's Logger's. Level is not showed in the output.
+
+```Ruby
+logger.error("This is a message")
+```
+
+Will output:
+```
+{ "message": "This is a message", "timestamp": "1970-01-01T00:00:00.000+0100" }
+```
+
+Timestamps use ISO8601
+Messages are json escaped so the total result is JSON parseable.
+
+
+### Adding keys to the output
+
+Keys can be added to the output at any moment. These keys will be output in each message till they are removed again. If you want to output keys in only one message use the _with_data method instead.
+
+Keys can be added using the `add_fields` method which accepts a hash:
+
+```Ruby
+logger.add_fields( 'role' => 'backend' )
+logger.error("This is a message")
+logger.warning("This is another message")
+```
+
+Will output:
+```json
+{ "message": "This is a message", "timestamp": "1970-01-01T00:00:00.000+0100", "role": "backend" }
+{ "message": "This is another message", "timestamp": "1970-01-01T00:00:00.000+0100", "role": "backend" }
+```
+
+Because of speed purposes the JSON dumping is done as simply as possible. If you provide a hash of keys like:
+```ruby
+{ key: 'value' }
+```
+The output will include:
+```json
+{ ":key": "value" }
+```
+
+
+### Logging methods with data
+
+All methods (info, debug, etc.) has a _with_data equivalent: "info_with_data", "debug_with_data".
+These methods accept and extra hash to add it to the JSON.
+
+```Ruby
+logger.error_with_data('message', {data1: 'Extra data', data2: 'Extra data2'})
+```
+
+Will output:
+```json
+{ "message": "This is a message", "timestamp": "1970-01-01T00:00:00.000+0100", "data": {"data1": "Extra data", "data2": "Extra data2"} }
+```
+
+
+### Logging exceptions
+
+There is a method to help you log exceptions in an standard way.
+
+```Ruby
+rescue StandardError => e
+  logger.exception(e)
+```
+
+Will output:
+```json
+{ "message": "#{e.message}", "timestamp": "1970-01-01T00:00:00.000+0100", "exception": "<exception name>", "stack": ["<stacktraceline1>", "<stacktraceline2>"] }
+```
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/lorekeeper.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,63 @@
+require 'benchmark'
+require 'tempfile'
+require 'securerandom'
+require 'benchmark/ips'
+require 'byebug'
+require 'rbtrace'
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __FILE__))
+$LOAD_PATH.uniq!
+
+require 'lorekeeper'
+require 'logger'
+
+def create_logger
+  logfile = Tempfile.new('my_test_log.log')
+  extra_fields = {
+    machine:"Verylongmachinenametobe-Pro.local",
+    component: 'Gilean', version: '0.1.1', trace_id: SecureRandom.hex(16), span_id: SecureRandom.hex(16), parent_span_id: SecureRandom.hex(16)
+  }
+  log = Lorekeeper::JSONLogger.new(logfile)
+  log.add_fields(extra_fields)
+  log
+end
+
+def create_simple_logger
+  logfile = Tempfile.new('my_test_log.log')
+  ::Logger.new(logfile.path)
+end
+
+# This task is used to help development of Lorekeeper. Use together with rbtrace
+desc "Runs the code once, sleeping to allow you to attach to it with rbtrace"
+task :run_once do
+  contents = 'This is a test, this is only a test. Do not worry about these contents.'
+  long_contents = contents * 100
+
+  log = create_logger
+  sleep(10)
+  log.error(long_contents)
+end
+
+
+# This task is used to help development of Lorekeeer. Make sure it is fast enough for your app.
+desc "Runs benchmarks for the library."
+task :benchmark do
+
+  contents = 'This is a test, this is only a test. Do not worry about these contents.'
+  long_contents = contents * 100
+
+  log = create_logger
+  simple_log = create_simple_logger
+
+  Benchmark.ips do |bm|
+    bm.report("short content") { log.error(contents) }
+    bm.report("Logger short content")  { simple_log.info(contents) }
+    bm.report("long content")  { log.info(long_contents) }
+    bm.report("Logger long content")   { simple_log.info(long_contents) }
+    bm.compare!
+  end
+
+  puts "i/s means the number of log messages written into a file per second"
+
+end
+

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "lorekeeper"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/lorekeeper.rb

@@ -0,0 +1,4 @@
+require 'lorekeeper/version'
+require 'lorekeeper/fast_logger'
+require 'lorekeeper/json_logger'
+require 'lorekeeper/multi_logger'

data/lib/lorekeeper/fast_logger.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+# The comment above will make all strings in a current file frozen
+
+require 'logger'
+
+module Lorekeeper
+
+  # Very simple, very fast logger
+  class FastLogger
+    include ::Logger::Severity  # contains the levels constants: DEBUG, ERROR, etc.
+    attr_accessor :level        # Current level, default: DEBUG
+    attr_accessor :formatter    # Just for compatibility with Logger, not used
+
+    def debug?; level <= DEBUG; end
+    def info?; level <= INFO; end
+    def warn?; level <= WARN; end
+    def error?; level <= ERROR; end
+    def fatal?; level <= FATAL; end
+
+    def initialize(file)
+      @level = DEBUG
+      @iodevice = LogDevice.new(file)
+    end
+
+    LOGGING_METHODS = %i(debug info warn error fatal)
+    METHOD_SEVERITY_MAP = {debug: DEBUG, info: INFO, warn: WARN, error: ERROR, fatal: FATAL}
+
+    # We define the behaviour of all the usual logging methods
+    # We support a string as a parameter and also a block
+    LOGGING_METHODS.each do |method_name|
+      define_method method_name.to_s, ->(message_param = nil, &block) do
+        return true if METHOD_SEVERITY_MAP[method_name] < @level
+        message = message_param || (block && block.call)
+        log_data(method_name, message.freeze)
+      end
+    end
+
+    # inherited classes probably want to reimplement this
+    def log_data(_method_sym, message)
+      @iodevice.write(message)
+    end
+
+    private
+
+    require 'monitor'
+    class LogDeviceMutex
+      include MonitorMixin
+    end
+
+    class LogDevice
+
+      def initialize(file)
+        @iodevice = to_iodevice(file)
+        @iomutex = LogDeviceMutex.new
+      end
+
+      def write(message)
+        return unless @iodevice
+        @iomutex.synchronize do
+          @iodevice.write(message)
+        end
+      end
+
+      private
+
+      def to_iodevice(file)
+        return nil unless file
+        iodevice = if file.respond_to?(:write) and file.respond_to?(:close)
+          file
+        else
+          open_logfile(file)
+        end
+        iodevice.sync = true if iodevice.respond_to?(:sync=)
+        iodevice
+      end
+
+      def open_logfile(filename)
+        File.open(filename, (File::WRONLY | File::APPEND))
+      rescue Errno::ENOENT => e
+        puts "File #{filename} can't be open for logging. #{e.message}"
+      end
+    end
+
+  end
+end

data/lib/lorekeeper/json_logger.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+# The comment above will make all strings in a current file frozen
+require 'oj'
+require 'lorekeeper/fast_logger'
+
+module Lorekeeper
+  # The JSONLogger provides a logger which will output messages in JSON format
+  class JSONLogger < FastLogger
+
+    def initialize(file)
+      reset_state
+      @base_fields = { MESSAGE => '', TIMESTAMP => '' }
+      super(file)
+    end
+
+    def current_fields
+      state[:base_fields]
+    end
+
+    def state
+      Thread.current[THREAD_KEY] ||= { base_fields: @base_fields.dup, extra_fields: {} }
+    end
+
+    def reset_state
+      Thread.current[THREAD_KEY] = nil
+    end
+
+    # Delegates methods to the existing Logger instance
+    # We are extending the logger API with methods error_with_data, etc
+    LOGGING_METHODS.each do |method_name|
+      define_method "#{method_name}_with_data", ->(message_param = nil, data = {}, &block) do
+        return true if METHOD_SEVERITY_MAP[method_name] < @level
+        extra_fields = {'data' => (data || {}) }
+        message = message_param || (block && block.call)
+        with_extra_fields(extra_fields) { log_data(method_name, message.freeze) }
+      end
+    end
+
+    def add_thread_unsafe_fields(fields)
+      remove_invalid_fields(fields)
+      @base_fields.merge!(fields)
+      reset_state # Forcing to recreate the thread safe information
+    end
+
+    def remove_thread_unsafe_fields(fields)
+      [*fields].each do |field|
+        @base_fields.delete(field)
+      end
+      reset_state
+    end
+
+    def add_fields(fields)
+      remove_invalid_fields(fields)
+      state.fetch(:base_fields).merge!(fields)
+    end
+
+    def remove_fields(fields)
+      [*fields].each do |field|
+        state.fetch(:base_fields).delete(field)
+      end
+    end
+
+    # @param exception: instance of a class inheriting from Exception
+    # We will output backtrace twice. Once inside the stack so it can be parsed by software
+    # And the other inside the message so it is readable to humans
+    def exception(exception, custom_message = nil, custom_data = nil)
+      if exception.is_a?(Exception)
+        backtrace = exception.backtrace || []
+        exception_fields = {
+          'exception' => "#{exception.class}: #{exception.message}",
+          'stack' => backtrace
+        }
+        exception_fields['data'] = custom_data if custom_data
+
+        message = custom_message || exception.message
+        with_extra_fields(exception_fields) { log_data(:error, message) }
+      else
+        log_data(:warning, 'Logger exception called without exception class.')
+        error_with_data("#{exception.class}: #{exception.inspect} #{custom_message}", custom_data)
+      end
+    end
+
+   private
+
+    THREAD_KEY = 'lorekeeper_jsonlogger_key'.freeze #Shared by all threads but unique by thread
+    MESSAGE = 'message'.freeze
+    TIMESTAMP = 'timestamp'.freeze
+    DATE_FORMAT = '%FT%T.%L%z'.freeze
+
+    def with_extra_fields(fields)
+      state[:extra_fields] = fields
+      yield
+      state[:extra_fields] = {}
+    end
+
+    def remove_invalid_fields(fields)
+      fields.delete_if do |_, v|
+        v.nil? || v.respond_to?(:empty?) && v.empty?
+      end
+    end
+
+    def log_data(_method_sym, message)
+      # merging is slow, we do not want to merge with empty hash if possible
+      fields_to_log = if state[:extra_fields].empty?
+        state[:base_fields]
+      else
+        state[:base_fields].merge(state[:extra_fields])
+      end
+
+      fields_to_log[MESSAGE] = message
+      fields_to_log[TIMESTAMP] = Time.now.utc.strftime(DATE_FORMAT)
+
+      @iodevice.write(Oj.dump(fields_to_log) + "\n")
+    end
+  end
+
+  # Simple logger which tries to have an easy to see output.
+  class SimpleLogger < FastLogger
+    SEVERITY_TO_COLOR_MAP   = {debug: '0;37', info: '0;37', warn: '33', error: '31', fatal: '31', unknown: '37'}
+
+    def log_data(method_sym, message)
+      color = SEVERITY_TO_COLOR_MAP[method_sym]
+      @iodevice.write("\033[#{color}m#{message}\033[0m\n")
+    end
+  end
+
+
+end

data/lib/lorekeeper/multi_logger.rb

@@ -0,0 +1,22 @@
+
+module Lorekeeper
+  # Allows to create a logger that will pass information to any logger registered
+  # It is useful so send the same message thought different loggers to different sinks
+  class MultiLogger
+    def initialize
+      @loggers = []
+    end
+    def add_logger(logger)
+      @loggers << logger
+    end
+
+    def method_missing(method, *args, &block)
+      result = @loggers.map do |logger|
+        logger.public_send(method, *args, &block) if logger.respond_to?(method)
+      end
+      # We call all the methods, delete nils and duplicates.
+      # Then hope for the best taking the first value
+      result.compact.uniq.first
+    end
+  end
+end

data/lib/lorekeeper/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+# The comment above will make all strings in a current file frozen
+module Lorekeeper
+  VERSION = '1.0.0'
+end

data/lorekeeper.gemspec

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+# The comment above will make all strings in a current file frozen
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'lorekeeper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "lorekeeper"
+  spec.version       = Lorekeeper::VERSION
+  spec.authors       = ["Jordi Polo"]
+  spec.email         = ["mumismo@gmail.com"]
+
+  spec.summary       = %q{ Very fast JSON logger }
+  spec.description   = %q{ Opinionated logger which outputs messages in JSON format }
+  spec.homepage      = "http://www.gihub.com/jordipolo/lorekeeper"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'oj', '~> 2.14'
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", '~> 3.4'
+  spec.add_development_dependency "benchmark-ips", '~> 2.3'
+  spec.add_development_dependency "timecop", '~> 0.8'
+  spec.add_development_dependency "byebug", '~> 8.0'
+  spec.add_development_dependency "rbtrace", '~> 0.4'
+
+end

metadata

@@ -0,0 +1,171 @@
+--- !ruby/object:Gem::Specification
+name: lorekeeper
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Jordi Polo
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-01-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '8.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '8.0'
+- !ruby/object:Gem::Dependency
+  name: rbtrace
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+description: " Opinionated logger which outputs messages in JSON format "
+email:
+- mumismo@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/lorekeeper.rb
+- lib/lorekeeper/fast_logger.rb
+- lib/lorekeeper/json_logger.rb
+- lib/lorekeeper/multi_logger.rb
+- lib/lorekeeper/version.rb
+- lorekeeper.gemspec
+homepage: http://www.gihub.com/jordipolo/lorekeeper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Very fast JSON logger
+test_files: []