scrolls 0.3.9 → 0.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 764cf377dd151093b60740032200866af5dfe033
-  data.tar.gz: ad73c8e48a7baa90366ca0e1a05eea887f719fb3
+SHA256:
+  metadata.gz: d70168aa379a644e25091bce26f7ce86e3ffb9d6564a97246d3e6251fbe353ad
+  data.tar.gz: 813b7fc23789ea3436e4f18f1be99f601ec034fbf737e70bfbf8705948676aed
 SHA512:
-  metadata.gz: 43df84fb536e8f94aa15e99471def2587ad5a8abe73ba018392d4aa9deacfff21eb1d2fbc11458670f43d459fe7a3cedeb190436bd5d93a997f7f2374fd35383
-  data.tar.gz: d565408e3abf2d46ce1f0561e9dbee59003f5041bbec07a0bdae3cd099295ec9cbf77c3bd8f2df97b1b15b797c1c7d452c1099df2d59ae53b444cb49e5c32424
+  metadata.gz: a419f6a9473eeaa2f7957db5d30ee5d1977149861542a7e28feb7e126f7413ee6a9876ff76e30414620fcf8a43eaa0abafe79408b23db0c4415a06f048ad5653
+  data.tar.gz: 3be6e7a630e21b35be17bf59ac480a40f2b98eec106c27dacfc245765574e317c680b5587a55064e5883b420c272ca1cdab687ac2052316fdbf53efdde7f9e45

data/Gemfile

@@ -4,5 +4,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :test do
+  gem "minitest", require: 'minitest/autorun'
+  gem "minitest-reporters"
   gem "rake"
 end

data/README.md

@@ -20,26 +20,65 @@ Or install it yourself as:
 
 Scrolls follows the belief that logs should be treated as data. One way to think of them is the blood of your infrastructure. Logs are a realtime view of what is happening on your systems.
 
-## Need to know!
+## Usage
+
+### 0.9.0 and later
 
-The way Scrolls handles "global_context" is changing after v0.3.8. Please see the [release notes](https://github.com/asenchi/scrolls/releases/tag/v0.3.8) and [this documentation](https://github.com/asenchi/scrolls/tree/master/docs/global-context.md) for more information. I apologize for any trouble this may cause.
+```ruby
+require 'scrolls'
 
-## Documentation:
+Scrolls.init(
+  timestamp: true,
+  global_context: {app: "scrolls", deploy: "production"},
+  exceptions: "multi"
+)
 
-I apologize, some of these are a WIP.
+Scrolls.log(at: "test")
 
-* [Sending logs to syslog using Scrolls](https://github.com/asenchi/scrolls/tree/master/docs/syslog.md)
-* Logging contexts
-* Adding timestamps by default
-* Misc Features
+Scrolls.context(context: "block") do
+  Scrolls.log(at: "exec")
+end
 
-## Usage
+begin
+  raise
+rescue Exception => e
+  Scrolls.log_exception(e, at: "raise")
+end
+```
+
+You can also use `Scrolls#log` and `Scrolls#log_exception` without initalizing:
+
+```ruby
+require 'scrolls'
+
+Scrolls.log(test: "test")
+```
+
+### Defaults
+
+Here are the defaults `Scrolls#init`:
+
+```
+stream: STDOUT
+facility: Syslog::LOG_USER
+time_unit: "seconds"
+timestamp: false
+exceptions: "single"
+global_context: {}
+syslog_options: Syslog::LOG_PID|Syslog::LOG_CONS
+escape_keys: false
+strict_logfmt: false
+```
+
+## Older Versions
+
+### Pre 0.9.0
 
 ```ruby
 require 'scrolls'
 
 Scrolls.add_timestamp = true
-Scrolls.global_context(:app => "scrolls", :deploy => ENV["DEPLOY"])
+Scrolls.global_context(:app => "scrolls", :deploy => "production")
 
 Scrolls.log(:at => "test")
 
@@ -57,10 +96,10 @@ end
 Produces:
 
 ```
-now="2014-01-17T16:11:39Z" app=scrolls deploy=nil at=test
-now="2014-01-17T16:11:39Z" app=scrolls deploy=nil context=block at=exec
-now="2014-01-17T16:11:39Z" app=scrolls deploy=nil at=exception class=RuntimeError message= exception_id=70312608019740
-now="2014-01-17T16:11:39Z" app=scrolls deploy=nil at=exception class= exception_id=70312608019740 site="./test.rb:16:in <main>"
+now="2017-09-01T00:37:13Z" app=scrolls deploy=production at=test
+now="2017-09-01T00:37:13Z" app=scrolls deploy=production context=block at=exec
+now="2017-09-01T00:37:13Z" app=scrolls deploy=production at=exception class=RuntimeError exception_id=70149797587080
+now="2017-09-01T00:37:13Z" app=scrolls deploy=production at=exception class=RuntimeError exception_id=70149797587080 site="./test-scrolls.rb:16:in <main>"
 ```
 
 ## History

data/Rakefile

@@ -1,11 +1,13 @@
 #!/usr/bin/env rake
+
 require "bundler/gem_tasks"
+require "rake/testtask"
 
 ENV['TESTOPTS'] = "-v"
 
-require "rake/testtask"
 Rake::TestTask.new do |t|
-  t.pattern = "test/test_*.rb"
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = true
 end
 
 task :default => :test

data/docs/global-context.md

@@ -38,7 +38,7 @@ Scrolls.init(
 Scrolls.log(:t => "t")
 ```
 
-Is the same as this currently:
+Is the same as this in versions prior to 0.9.0:
 
 ```ruby
 Scrolls.global_context(:g => "g")

data/docs/syslog.md

@@ -6,6 +6,14 @@ By default Scrolls writes log messages to `STDOUT`. With the release of [v0.2.8]
 Scrolls.stream = "syslog"
 ```
 
+Or using `Scrolls#init` in versions 0.9.0 and after:
+
+```ruby
+Scrolls.init(
+    stream: "syslog"
+)
+```
+
 This defaults to syslog facility USER and log level ERROR. You can adjust the log facility like so:
 
 ```ruby
@@ -14,4 +22,9 @@ Scrolls.facility = "local7"
 
 Scrolls generally doesn't care about log levels. The library defaults to ERROR (or 3), but ultimately is of the opinion that levels are useless. The reasoning behind this is that applications should log useful data, all of the time. Debugging data is great for development, but should never be deployed. The richness of structured logging allows exceptions and error messages to sit along side the context of the data in which the error was thrown, there is no need to send to an "emergency" level.
 
-With that said, if one wanted to adjust the log level, you can set an environment variable `LOG_LEVEL`. This allows this particular feature to be rather fluid throughout your application.
+With that said, if one wanted to adjust the log level, you can set an environment variable `LOG_LEVEL` or use one of the level methods. This allows this particular feature to be rather fluid throughout your application.
+
+```ruby
+Scrolls.info(d: "data")
+Scrolls.warn(d: "data")
+```

data/lib/scrolls/{iolog.rb → iologger.rb}

@@ -1,7 +1,9 @@
 module Scrolls
-  class IOLog
+  class IOLogger
     def initialize(stream)
-      stream.sync = true
+      if stream.respond_to?(:sync)
+        stream.sync = true
+      end
       @stream = stream
     end
 

data/lib/scrolls/logger.rb

@@ -0,0 +1,323 @@
+require "syslog"
+
+require "scrolls/parser"
+require "scrolls/iologger"
+require "scrolls/sysloglogger"
+require "scrolls/utils"
+
+module Scrolls
+  # Default log facility
+  LOG_FACILITY = ENV['LOG_FACILITY'] || Syslog::LOG_USER
+
+  # Default log level
+  LOG_LEVEL = (ENV['LOG_LEVEL'] || 6).to_i
+
+  # Default syslog options
+  SYSLOG_OPTIONS = Syslog::LOG_PID|Syslog::LOG_CONS
+
+  class TimeUnitError < RuntimeError; end
+  class LogLevelError < StandardError; end
+
+  # Top level class to hold our global context
+  #
+  # Global context is defined using Scrolls#init
+  class GlobalContext
+    def initialize(ctx)
+      @ctx = ctx || {}
+    end
+
+    def to_h
+      @ctx
+    end
+  end
+
+  class Logger
+
+    attr_reader :logger
+    attr_accessor :exceptions, :timestamp
+
+    def initialize(options={})
+      @stream        = options.fetch(:stream, STDOUT)
+      @log_facility  = options.fetch(:facility, LOG_FACILITY)
+      @time_unit     = options.fetch(:time_unit, "seconds")
+      @timestamp     = options.fetch(:timestamp, false)
+      @exceptions    = options.fetch(:exceptions, "single")
+      @global_ctx    = options.fetch(:global_context, {})
+      @syslog_opts   = options.fetch(:syslog_options, SYSLOG_OPTIONS)
+      @escape_keys   = options.fetch(:escape_keys, false)
+      @strict_logfmt = options.fetch(:strict_logfmt, false)
+
+      # Our main entry point to ensure our options are setup properly
+      setup!
+    end
+
+    def context
+      if Thread.current.thread_variables.include?(:scrolls_context)
+        Thread.current.thread_variable_get(:scrolls_context)
+      else
+        Thread.current.thread_variable_set(:scrolls_context, {})
+      end
+    end
+
+    def context=(h)
+      Thread.current.thread_variable_set(:scrolls_context, h || {})
+    end
+
+    def stream
+      @stream
+    end
+
+    def stream=(s)
+      # Return early to avoid setup
+      return if s == @stream
+
+      @stream = s
+      setup_stream
+    end
+
+    def escape_keys?
+      @escape_keys
+    end
+
+    def strict_logfmt?
+      @strict_logfmt
+    end
+
+    def syslog_options
+      @syslog_opts
+    end
+
+    def facility
+      @facility
+    end
+
+    def facility=(f)
+      if f
+        setup_facility(f)
+        # If we are using syslog, we need to setup our connection again
+        if stream == "syslog"
+          @logger = Scrolls::SyslogLogger.new(
+                      progname,
+                      syslog_options,
+                      facility
+                    )
+        end
+      end
+    end
+
+    def time_unit
+      @time_unit
+    end
+
+    def time_unit=(u)
+      @time_unit = u
+      setup_time_unit
+    end
+
+    def global_context
+      @global_context.to_h
+    end
+
+    def log(data, &blk)
+      # If we get a string lets bring it into our structure.
+      if data.kind_of? String
+        rawhash = { "log_message" => data }
+      else
+        rawhash = data
+      end
+
+      if gc = @global_context.to_h
+        ctx = gc.merge(context)
+        logdata = ctx.merge(rawhash)
+      end
+
+      # By merging the logdata into the timestamp, rather than vice-versa, we
+      # ensure that the timestamp comes first in the Hash, and is placed first
+      # on the output, which helps with readability.
+      logdata = { :now => Time.now.utc }.merge(logdata) if prepend_timestamp?
+
+      unless blk
+        write(logdata)
+      else
+        start = Time.now
+        res = nil
+        log(logdata.merge(:at => "start"))
+        begin
+          res = yield
+        rescue StandardError => e
+          logdata.merge!({
+            at:           "exception",
+            reraise:      true,
+            class:        e.class,
+            message:      e.message,
+            exception_id: e.object_id.abs,
+            elapsed:      calculate_time(start, Time.now)
+          })
+          logdata.delete_if { |k,v| k if v == "" }
+          log(logdata)
+          raise e
+        end
+        log(logdata.merge(:at => "finish", :elapsed => calculate_time(start, Time.now)))
+        res
+      end
+    end
+
+    def log_exception(e, data=nil)
+      unless @defined
+        @stream = STDERR
+        setup_stream
+      end
+
+      # We check our arguments for type
+      case data
+      when String
+        rawhash = { "log_message" => data }
+      when Hash
+        rawhash = data
+      else
+        rawhash = {}
+      end
+
+      if gc = @global_context.to_h
+        logdata = gc.merge(rawhash)
+      end
+
+      excepdata = {
+        at:           "exception",
+        class:        e.class,
+        message:      e.message,
+        exception_id: e.object_id.abs
+      }
+
+      excepdata.delete_if { |k,v| k if v == "" }
+
+      if e.backtrace
+        if single_line_exceptions?
+          lines = e.backtrace.map { |line| line.gsub(/[`'"]/, "") }
+
+          if lines.length > 0
+            excepdata[:site] = lines.join('\n')
+            log(logdata.merge(excepdata))
+          end
+        else
+          log(logdata.merge(excepdata))
+
+          e.backtrace.each do |line|
+            log(logdata.merge(excepdata).merge(
+              :at           => "exception",
+              :class        => e.class,
+              :exception_id => e.object_id.abs,
+              :site         => line.gsub(/[`'"]/, "")
+            ))
+          end
+        end
+      end
+    end
+
+    def with_context(prefix)
+      return unless block_given?
+      old = context
+      self.context = old.merge(prefix)
+      res = yield if block_given?
+    ensure
+      self.context = old
+      res
+    end
+
+    private
+
+    def setup!
+      setup_global_context
+      prepend_timestamp?
+      setup_facility
+      setup_stream
+      single_line_exceptions?
+      setup_time_unit
+    end
+
+    def setup_global_context
+      # Builds up an immutable object for our global_context
+      # This is not backwards compatiable and was introduced after 0.3.7.
+      # Removes ability to add to global context once we initialize our
+      # logging object. This also deprecates #add_global_context.
+      @global_context = GlobalContext.new(@global_ctx)
+      @global_context.freeze
+    end
+
+    def prepend_timestamp?
+      @timestamp
+    end
+
+    def setup_facility(f=nil)
+      if f
+        @facility = LOG_FACILITY_MAP.fetch(f, LOG_FACILITY)
+      else
+        @facility = LOG_FACILITY_MAP.fetch(@log_facility, LOG_FACILITY)
+      end
+    end
+
+    def setup_stream
+      unless @stream == STDOUT
+        # Set this so we know we aren't using our default stream
+        @defined = true
+      end
+
+      if @stream == "syslog"
+        @logger = Scrolls::SyslogLogger.new(
+                    progname,
+                    syslog_options,
+                    facility
+                  )
+      else
+        @logger = IOLogger.new(@stream)
+      end
+    end
+
+    def single_line_exceptions?
+      return false if @exceptions == "multi"
+      true
+    end
+
+    def setup_time_unit
+      unless %w{s ms seconds milliseconds}.include? @time_unit
+        raise TimeUnitError, "Specify the following: s, ms, seconds, milliseconds"
+      end
+
+      case @time_unit
+      when %w{s seconds}
+        @t = 1.0
+      when %w{ms milliseconds}
+        @t = 1000.0
+      else
+        @t = 1.0
+      end
+    end
+
+    # We need this for our syslog setup
+    def progname
+      File.basename($0)
+    end
+
+    def calculate_time(start, finish)
+      translate_time_unit unless @t
+      ((finish - start).to_f * @t)
+    end
+
+    def log_level_ok?(level)
+      if level
+        raise LogLevelError, "Log level unknown" unless LOG_LEVEL_MAP.key?(level)
+        LOG_LEVEL_MAP[level.to_s] <= LOG_LEVEL
+      else
+        true
+      end
+    end
+
+    def write(data)
+      if log_level_ok?(data[:level])
+        msg = Scrolls::Parser.unparse(data, escape_keys=escape_keys?, strict_logfmt=strict_logfmt?)
+        @logger.log(msg)
+      end
+    end
+
+  end
+end

data/lib/scrolls/parser.rb

@@ -4,8 +4,10 @@ module Scrolls
   module Parser
     extend self
 
-    def unparse(data)
+    def unparse(data, escape_keys=false, strict_logfmt=false)
       data.map do |(k,v)|
+        k = Scrolls::Utils.escape_chars(k) if escape_keys
+
         if (v == true)
           "#{k}=true"
         elsif (v == false)
@@ -21,7 +23,7 @@ module Scrolls
           has_single_quote = v.index("'")
           has_double_quote = v.index('"')
           if v =~ /[ =:,]/
-            if has_single_quote && has_double_quote
+            if (has_single_quote || strict_logfmt) && has_double_quote
               v = '"' + v.gsub(/\\|"/) { |c| "\\#{c}" } + '"'
             elsif has_double_quote
               v = "'" + v.gsub('\\', '\\\\\\') + "'"

data/lib/scrolls/sysloglogger.rb

@@ -0,0 +1,17 @@
+module Scrolls
+  class SyslogLogger
+    def initialize(ident = 'scrolls',
+                   options = Scrolls::SYSLOG_OPTIONS,
+                   facility = Scrolls::LOG_FACILITY)
+      if Syslog.opened?
+        @syslog = Syslog.reopen(ident, options, facility)
+      else
+        @syslog = Syslog.open(ident, options, facility)
+      end
+    end
+
+    def log(data)
+      @syslog.log(Syslog::LOG_INFO, "%s", data)
+    end
+  end
+end

data/lib/scrolls/utils.rb

@@ -1,19 +1,61 @@
 module Scrolls
-  module Utils
 
-    def hashify(d)
-      last = d.pop
-      return {} unless last
-      return hashified_list(d).merge(last) if last.is_a?(Hash)
-      d.push(last)
-      hashified_list(d)
-    end
+  # Helpful map of syslog facilities
+  LOG_FACILITY_MAP = {
+    "auth"     => Syslog::LOG_AUTH,
+    "authpriv" => Syslog::LOG_AUTHPRIV,
+    "cron"     => Syslog::LOG_CRON,
+    "daemon"   => Syslog::LOG_DAEMON,
+    "ftp"      => Syslog::LOG_FTP,
+    "kern"     => Syslog::LOG_KERN,
+    "mail"     => Syslog::LOG_MAIL,
+    "news"     => Syslog::LOG_NEWS,
+    "syslog"   => Syslog::LOG_SYSLOG,
+    "user"     => Syslog::LOG_USER,
+    "uucp"     => Syslog::LOG_UUCP,
+    "local0"   => Syslog::LOG_LOCAL0,
+    "local1"   => Syslog::LOG_LOCAL1,
+    "local2"   => Syslog::LOG_LOCAL2,
+    "local3"   => Syslog::LOG_LOCAL3,
+    "local4"   => Syslog::LOG_LOCAL4,
+    "local5"   => Syslog::LOG_LOCAL5,
+    "local6"   => Syslog::LOG_LOCAL6,
+    "local7"   => Syslog::LOG_LOCAL7,
+  }
+
+  # Helpful map of syslog log levels
+  LOG_LEVEL_MAP = {
+    "emerg"     => 0, # Syslog::LOG_EMERG
+    "emergency" => 0, # Syslog::LOG_EMERG
+    "alert"     => 1, # Syslog::LOG_ALERT
+    "crit"      => 2, # Syslog::LOG_CRIT
+    "critical"  => 2, # Syslog::LOG_CRIT
+    "error"     => 3, # Syslog::LOG_ERR
+    "warn"      => 4, # Syslog::LOG_WARNING
+    "warning"   => 4, # Syslog::LOG_WARNING
+    "notice"    => 5, # Syslog::LOG_NOTICE
+    "info"      => 6, # Syslog::LOG_INFO
+    "debug"     => 7  # Syslog::LOG_DEBUG
+  }
+
+  ESCAPE_CHAR = {
+    "&" => "&",
+    "<" => "&lt;",
+    ">" => "&gt;",
+    "'" => "&#x27;",
+    '"' => "&quot;",
+    "/" => "&#x2F;"
+  }
+
+  ESCAPE_CHAR_PATTERN = Regexp.union(*ESCAPE_CHAR.keys)
+
+  module Utils
 
-    def hashified_list(l)
-      return {} if l.empty?
-      l.inject({}) do |h, i|
-        h[i.to_sym] = true
-        h
+    def self.escape_chars(d)
+      if d.is_a?(String) and d =~ ESCAPE_CHAR_PATTERN
+        esc = d.to_s.gsub(ESCAPE_CHAR_PATTERN) {|c| ESCAPE_CHAR[c] }
+      else
+        esc = d
       end
     end
 

data/lib/scrolls/version.rb

@@ -1,3 +1,3 @@
 module Scrolls
-  VERSION = "0.3.9"
+  VERSION = "0.9.3"
 end