smarter_logging 1.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e793102aedb16d70480b3b012d9a220973f7ae4d
+  data.tar.gz: 11d8c0d59dc0797adfb334a54f21615bd127eb14
+SHA512:
+  metadata.gz: 16d8b4cb389c2b97fd84fffb14ba0dc70b2daca77510b76eae75aa9eb2d3a71ddcbf5e81982ce50ac48531a100c477ea12453185ced00ab887a4b5711926d3fd
+  data.tar.gz: f2e0972a840b18c442270632f4601b798a8752bd2308cebd74bba1fcea767f0ae83dd7a5936437b502abfb109ad326c302ea78905d8d8a42dd7d7180994ef372

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+*~
+*.bak

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.3
+before_install: gem install bundler -v 1.14.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in smarter_logging.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Tilo Sloboda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,126 @@
+# SmarterLogging
+
+SmarterLogging helps you log mission critical data in key=value and single-line format, so it can be easily analyzed.
+The format lends itself to being easily importable by SumoLogic and Splunk.
+
+The main idea is to have dedicated log files for `activities` which happen during normal operations, as well as `anomalies` which happen when something unexpected happens.
+
+Each log line starts with a UTC timestamp in [ISO8601 format](https://en.wikipedia.org/wiki/ISO_8601), and contains space-separated key=value pairs. If a value contains spaces, it is quoted: `key="value"`.
+The advantage of ISO6801 timestamps is that they are both standardized / easy to parse, as well as human readable.
+
+      time=2017-01-30T21:21:04.013Z activity=some_unique_name key1=value1 key2="value 2"
+      
+The only place where this gem is opinionated is that UTC time is used for logging.
+
+**Tip:** If you are currently not using UTC time in your application, you should seriously consider doing that - it makes it much easier analyzing logs and errors when dealing with clients which can be anywhere in the world.
+
+Key goals:
+
+* get average call durations
+* get activity and anomaly logs
+* help forensics on any calls / trace incidents 
+* quickly identify top anomalies
+* get timings for blocks of code
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+
+    gem 'smarter_logging'
+
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install smarter_logging
+
+## Usage
+
+When included in a Rails project, two default loggers are defined: 
+
+    SmarterLogging.anomaly_logger
+    SmarterLogging.activity_logger
+
+Each of them provides a `log()` method. 
+
+As a shortcut, you can `include SmarterLogging` and do this:
+
+    SmarterLogging.log_anomaly(unique_key, params)
+    SmarterLogging.log_activity(unique_key, params)
+
+You can either use the above two functioncalls throughout your code, or include the module `SmarterLogging` and use the methods directly throughout your code.
+
+## Examples
+
+**Tip:** make sure that you use a globally unique naming convention for your anomaly and activity names -- this way you can pin-point what went wrong, and where it went wrong.
+
+### Logging Anomalies
+
Anomaly logs are meant to be used for errors, exceptions and caught unexpected behavior.
+
+      include SmarterLogging
+      
      # you can use a single-line log statement, with a hash of all key=value pairs
+      log_anomaly( :invalid_parameter , {key1: 'value 1', key2: 'value 2'}     
+      
+      # or you can use a block - this will add a key `duration` to the log line.
+
+
+      if parameters_valid?(params)
+      
+        # do something useful
+        
+        log_activity( :user_parameters, params )
+        
+      else 
+        log_anomaly( :user_invalid_parameters ) do |logdata|		    
+           log_data.merge( invalid_parameter_hash )
+
+			 # some other code      	  	
+			 
+        end
+      end
+
+      
+      
+### Logging Activities
+
+Anomaly logs are meant to be used for reporting on expected behavior.
+
+      include SmarterLogging
+      
+      # you can use a single-line log statement, with a hash of all key=value pairs
+      log_activity( :user_signed_up, {user_id: current_user.id} )
+      => time=2017-01-30T23:15:20.689Z env=development activity=user_signed_up user_id=123
+      
+      # or you can use a block - this will add a key `duration` to the log line.
+      log_activity( :user_updated ) do |logdata|
+	     begin
+	     
+          # update some values in user record here
+          
+          logdata[:result] = :sucess
+          
        rescue => e
+    	    log_data[:error_message] = e.message
+      	  	
+           logdata[:result] = :failure
+        end
+      end
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/Tilo Sloboda/smarter_logging. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "smarter_logging"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/smarter_logging/activity_logger.rb

@@ -0,0 +1,13 @@
+module SmarterLogging
+
+  class ActivityLogger < BaseLogger
+    attr_accessor :anomaly_logger # to keep a reference to the anomaly logger
+
+    def log(unique_identifier, data, &block)
+      data = {:activity => unique_identifier}.merge( data )
+      _log_wrapper(data, &block)
+    end
+
+  end
+
+end

data/lib/smarter_logging/anomaly_logger.rb

@@ -0,0 +1,17 @@
+module SmarterLogging
+
+  class AnomalyLogger < BaseLogger
+
+    def log(unique_identifier, data, &block)
+      data = {:anomaly => unique_identifier}.merge( data )
+      _log_wrapper(data, &block)
+    end
+
+    def _log(data)
+      # If activity logging broke, make sure we log it as an anomaly:
+      data[:anomaly] = data.delete(:activity) if data[:activity]
+      super
+    end
+  end
+
+end

data/lib/smarter_logging/base_logger.rb

@@ -0,0 +1,89 @@
+require 'fileutils'
+
+module SmarterLogging
+  class BaseLogger
+
+    def initialize(log_filename)
+      if defined?(Rails)
+        log_filename = "#{Rails.root}/log/#{log_filename}"
+      else
+        FilUtils.mkdir './log'
+        log_filename = "./log/#{log_filename}"
+      end
+      @logger = Logger.new(log_filename)
+      @logger.formatter = nil if @logger.respond_to?(:formatter)
+    end
+
+    # this wrapper around _log() handles blocks and errors which could handle in a block
+    # if there is an exception during evaluation of the block, we log it as an anomaly and re-raise the exception
+    def _log_wrapper(data={}, &block)
+      had_anomaly = false
+      if block_given?
+        begin
+          start = Time.now.utc  # to measure duration of the block
+          result = yield(data)
+          data[:duration] = ((Time.now.utc - start) * 1000).to_f.round(3) # in ms
+          result
+
+        rescue => e
+          data[:exception] = e
+          data[:duration] = ((Time.now.utc - start) * 1000).to_f.round(3) # in ms
+
+          # we should log this to the anomaly logger also if one is defined
+          if @anomaly_logger
+            @anomaly_logger._log( data )
+            had_anomaly = true
+          else
+            data[:success] = false
+          end
+          raise # re-raise
+        ensure
+          # log it, even if there is no anomaly logger
+          _log( data ) unless had_anomaly  # if it was logged as an anomaly, don't log again
+        end
+
+      else # without a block, just call the lowest-level logging
+        _log( data )
+      end
+    end
+
+    # lowest-level logging method:
+    def _log(data)
+      if exception = data.delete(:exception)
+        data[:error] = [exception.class, exception.message].join[' : ']
+        data[:backtrace] = exception.backtrace[0..5].join(' | ')
+      end
+
+      extra_data = {
+        :time => Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.%LZ"),
+        :env => Rails.env,
+      }
+      SmarterLogging::ControllerHelper::KEYS.each do |key|
+        extra_data[key] = Thread.current[key] if Thread.current[key]
+      end
+
+      data = extra_data.merge( data )
+      # using low-level logging:
+      @logger << reformat( data ) + "\n"
+    end
+
+    private
+    # reformat incoming values, and make sure they are not listed in the Rails Filter Parameters
+    def reformat(data)
+      filtered_keys = Rails.application.config.filter_parameters.map{|x| x.to_s}
+      result = []
+      data.each do |k,v|
+        if filtered_keys.include?(k.to_s)
+          v = '[FILTERED]'
+        elsif v =~ /\s/  # quote values with spaces
+          v = "\"#{v}\""
+        end
+
+        result << "#{k}=#{v}"
+      end
+      result.join(' ')
+    end
+
+  end
+
+end

data/lib/smarter_logging/controller_helper.rb

@@ -0,0 +1,21 @@
+module SmarterLogging
+
+  module ControllerHelper
+    KEYS = [:rqid, :ip, :user_agent]
+
+    def self.included(base)
+      base.class_eval do
+        before_filter do
+          # Get some basic info from the current thread, and add it to the log parameters:
+          # The RequestID RQID can be passed-in on subsequent API-calls,
+          # so if several APIs call each other, they can be traced with the initial call's RQID
+          Thread.current[:rqid] = request.headers['X_RQID'] || ::SmarterLogging::Extensions::String.random(24)
+          Thread.current[:ip] = request.remote_ip || request.ip
+          Thread.current[:user_agent] = request.user_agent
+        end
+      end
+    end
+
+  end
+
+end

data/lib/smarter_logging/extensions/string.rb

@@ -0,0 +1,36 @@
+module SmarterLogging
+  module Extensions
+
+    module String # adding functionality without monkey patching String
+
+      # Create a random String of given length, using given character set
+      #
+      # Character set is an Array which can contain Ranges, Arrays, Characters
+      #
+      # Examples
+      #
+      #     String.random
+      #     String.random(10, ['0'..'9','A'..'F'] )
+      #     => "3EBF48AD3D"
+      #
+      #     BASE64_CHAR_SET =  ["A".."Z", "a".."z", "0".."9", '_', '-']
+      #     String.random(10, BASE64_CHAR_SET)
+      #     => "xM_1t3qcNn"
+      #
+      #     SPECIAL_CHARS = ["!", "@", "#", "$", "%", "^", "&", "*", "(", ")", "-", "_", "=", "+", "|", "/", "?", ".", ",", ";", ":", "~", "`", "[", "]", "{", "}", "<", ">"]
+      #     BASE91_CHAR_SET =  ["A".."Z", "a".."z", "0".."9", SPECIAL_CHARS]
+      #     String.random(10, BASE91_CHAR_SET)
+      #      => "S(Z]z,J{v;"
+      #
+      # CREDIT: Tilo Sloboda
+      #
+      # SEE: https://gist.github.com/tilo/3ee8d94871d30416feba
+
+      def self.random(len=32, character_set = ["A".."Z", "a".."z", "0".."9"])
+        chars = character_set.map{|x| x.is_a?(Range) ? x.to_a : x }.flatten
+        Array.new(len){ chars.sample }.join
+      end
+
+    end
+  end
+end

data/lib/smarter_logging/railtie.rb

@@ -0,0 +1,27 @@
+module SmarterLogging
+  mattr_accessor :anomaly_logger
+  mattr_accessor :activity_logger
+
+  class Railtie < Rails::Railtie
+    initializer 'smarter_logging.configure_rails_initialization' do |app|
+      app.config.anomaly_logger = SmarterLogging.anomaly_logger = SmarterLogging::AnomalyLogger.new('anomaly.log')
+      # make sure the activity logger has a reference to the anomaly logger, in case errors need to be logged:
+      act_logger = SmarterLogging::ActivityLogger.new('activity.log')
+      act_logger.anomaly_logger = SmarterLogging.anomaly_logger
+
+      app.config.activity_logger = SmarterLogging.activity_logger = act_logger
+    end
+  end
+
+  # after including SmarterLogging, you can simply use these convenience methods:
+
+  def self.log_activity(unique_identifier, data, &block)
+    activity_logger.log(unique_identifier, data, &block)
+  end
+
+  def self.log_anomaly(unique_identifier, data, &block)
+    anomaly_logger.log(unique_identifier, data, &block)
+  end
+
+end
+

data/lib/smarter_logging/version.rb

@@ -0,0 +1,3 @@
+module SmarterLogging
+  VERSION = "1.0.4"
+end

data/lib/smarter_logging.rb

@@ -0,0 +1,15 @@
+require 'smarter_logging/version'
+
+require 'smarter_logging/extensions/string'
+
+require 'smarter_logging/base_logger'
+require 'smarter_logging/anomaly_logger'
+require 'smarter_logging/activity_logger'
+require 'smarter_logging/controller_helper'
+
+require 'smarter_logging/railtie' if defined?(Rails)
+
+
+module SmarterLogging
+  # Your code goes here...
+end

data/smarter_logging.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'smarter_logging/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "smarter_logging"
+  spec.version       = SmarterLogging::VERSION
+  spec.authors       = ["Tilo Sloboda"]
+  spec.email         = ["tilo.sloboda@gmail.com"]
+
+  spec.summary       = %q{Ruby Gem for smarter logging of key=value data}
+  spec.description   = %q{Ruby Gem for smarter logging of key=value data in single-line format}
+  spec.homepage      = "https://github.com/tilo/smarter_logging"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: smarter_logging
+version: !ruby/object:Gem::Version
+  version: 1.0.4
+platform: ruby
+authors:
+- Tilo Sloboda
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-02-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Ruby Gem for smarter logging of key=value data in single-line format
+email:
+- tilo.sloboda@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/smarter_logging.rb
+- lib/smarter_logging/activity_logger.rb
+- lib/smarter_logging/anomaly_logger.rb
+- lib/smarter_logging/base_logger.rb
+- lib/smarter_logging/controller_helper.rb
+- lib/smarter_logging/extensions/string.rb
+- lib/smarter_logging/railtie.rb
+- lib/smarter_logging/version.rb
+- smarter_logging.gemspec
+homepage: https://github.com/tilo/smarter_logging
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Ruby Gem for smarter logging of key=value data
+test_files: []