lines 0.1.0

data/README.md

@@ -0,0 +1,18 @@
+Lines - structured logs for humans
+==================================
+
+An oppinionated logging library that implement the
+[lines](https://github.com/zimbatm/lines) format.
+
+Status: work in progress
+
+TODO
+====
+
+Add context to the library
+
+Check performances
+
+Exception formatting
+
+Fix the UniqueID algorithm

data/lib/lines.rb

@@ -0,0 +1,343 @@
+require 'date'
+require 'time'
+require 'forwardable'
+
+# Lines is an opinionated structured log format and a library
+# inspired by Slogger.
+#
+# Don't use log levels. They limit the reasoning of the developer.
+# Log everything in development AND production.
+# Logs should be easy to read, grep and parse.
+# Logging something should never fail.
+# Use syslog.
+#
+# Example:
+#
+#     log(msg: "Oops !")
+#     #outputs:
+#     # at=2013-03-07T09:21:39+00:00 pid=3242 app=some-process msg="Oops !" foo={} g=[]
+#
+# Usage:
+#
+#     Lines.use(Syslog, $stderr)
+#     Lines.log(foo: 3, msg: "This")
+#
+#     ctx = Lines.context(encoding_id: Log.id)
+#     ctx.log({})
+#
+#     Lines.context(:foo => :bar) do |l|
+#       l.log(:sadfasdf => 3)
+#     end
+module Lines
+  # New lines in Lines
+  NL = "\n".freeze
+
+  @global = {}
+  @outputters = []
+
+  class << self
+    def dumper; @dumper ||= Dumper.new end
+    attr_reader :global
+    attr_reader :outputters
+
+    # Used to select what output the lines will be put on.
+    #
+    # outputs - allows any kind of IO or Syslog
+    #
+    # Usage:
+    #
+    #     Lines.use(Syslog, $stderr)
+    def use(*outputs)
+      outputters.replace(outputs.map{|o| to_outputter o})
+    end
+
+    # The main function. Used to record objects in the logs as lines.
+    #
+    # obj - a ruby hash
+    # args -
+    def log(obj, args={})
+      obj = sanitize_obj(obj, args)
+      obj = global.merge(obj)
+      outputters.each{|out| out.output(dumper, obj) }
+      obj
+    end
+
+    # Add data to the logs
+    #
+    # data - a ruby hash
+    def context(data={})
+      new_context = Context.new global.merge(data)
+      yield new_context if block_given?
+      new_context
+    end
+
+    class Context
+      attr_reader :data
+
+      def initialize(data)
+        @data = data
+      end
+
+      def log(obj, args={})
+        Lines.log(obj, args.merge(data))
+      end
+    end
+
+    # A backward-compatibile logger
+    def logger
+      @logger ||= (
+        require "lines/logger"
+        Logger.new(self)
+      )
+    end
+
+    protected
+
+    def sanitize_obj(obj, args={})
+      if obj.kind_of?(Exception)
+        ex = obj
+        obj = {ex: ex.class, msg: ex.to_s}
+        if ex.respond_to?(:backtrace) && ex.backtrace
+          obj[:backtrace] = ex.backtrace
+        end
+      elsif !obj.kind_of?(Hash)
+        if obj.respond_to?(:to_h)
+          obj = obj.to_h
+        else
+          obj = {msg: obj}
+        end
+      end
+      obj.merge(args)
+    end
+
+    def to_outputter(out)
+      return out if out.respond_to?(:output)
+      return StreamOutputter.new(out) if out.respond_to?(:write)
+
+      case out
+      when IO
+        StreamOutputter.new(out)
+      when Syslog
+        SyslogOutputter.new
+      else
+        raise ArgumentError, "unknown outputter #{out.inspect}"
+      end
+    end
+  end
+
+  class StreamOutputter
+    # stream must accept a #write(str) message
+    def initialize(stream = $stderr)
+      @stream = stream
+      # Is this needed ?
+      @stream.sync = true if @stream.respond_to?(:sync)
+    end
+
+    def output(dumper, obj)
+      str = dumper.dump(obj) + NL
+      stream.write str
+    end
+
+    protected
+
+    attr_reader :stream
+  end
+
+  require 'syslog'
+  class SyslogOutputter
+    PRI2SYSLOG = {
+      debug:    Syslog::LOG_DEBUG,
+      info:     Syslog::LOG_INFO,
+      warn:     Syslog::LOG_WARNING,
+      warning:  Syslog::LOG_WARNING,
+      err:      Syslog::LOG_ERR,
+      error:    Syslog::LOG_ERR,
+      crit:     Syslog::LOG_CRIT,
+      critical: Syslog::LOG_CRIT,
+    }
+
+    def initialize(syslog = Syslog, app_name=nil)
+      @app_name = app_name
+      @syslog = syslog
+      prepare_syslog
+    end
+
+    def output(dumper, obj)
+      obj = obj.dup
+      obj.delete(:pid) # It's going to be part of the message
+      obj.delete(:at)  # Also part of the message
+      obj.delete(:app) # And again
+
+      level = extract_pri(obj)
+      str = dumper.dump(obj)
+
+      syslog.log(level, str)
+    end
+
+    protected
+
+    attr_reader :app_name
+    attr_reader :syslog
+
+    def prepare_syslog
+      unless syslog.opened?
+        # Did you know ? app_name is detected by syslog if nil
+        syslog.open(app_name,
+                    Syslog::LOG_PID & Syslog::LOG_CONS & Syslog::LOG_NDELAY,
+                    Syslog::LOG_USER)
+      end
+    end
+
+    def extract_pri(h)
+      pri = h.delete(:pri).to_s.downcase
+      PRI2SYSLOG[pri] || PRI2SYSLOG[:info]
+    end
+  end
+
+  # Some opinions here as well on the format:
+  #
+  # We really want to never fail at dumping because you know, they're logs.
+  # It's better to get a slightly less readable log that no logs at all.
+  #
+  # We're trying to be helpful for humans. It means that if possible we want
+  # to make things shorter and more readable. It also means that ideally
+  # we would like the parsing to be isomorphic but approximations are alright.
+  # For example a symbol might become a string.
+  #
+  # Basically, values are either composite (dictionaries and arrays), quoted
+  # strings or litterals. Litterals are strings that can be parsed to
+  # something else depending if the language supports it or not.
+  # Litterals never contain white-spaces or other weird (very precise !) characters.
+  #
+  # the true litteral is written as "#t"
+  # the false litteral is written as "#f"
+  # the nil / null litteral is written as "nil"
+  #
+  # dictionary keys are always strings or litterals.
+  #
+  # Pleaaase, keep units with numbers. And we provide a way for this:
+  # a tuple of (number, litteral) can be concatenated. Eg: (3, 'ms') => 3ms
+  # alternatively if your language supports a time range it could be serialized
+  # to the same value (and parsed back as well).
+  #
+  # if we don't know how to serialize something we provide a language-specific
+  # string of it and encode is at such.
+  #
+  # The output ought to use the UTF-8 encoding.
+  #
+  # This dumper has been inspired by the OkJSON gem (both formats look alike
+  # after all).
+  class Dumper
+    def dump(obj) #=> String
+      objenc_internal(obj)
+    end
+
+    # Used to introduce new ruby litterals.
+    def map(klass, &rule)
+      @mapping[klass] = rule
+    end
+
+    protected
+
+    attr_reader :mapping
+
+    def initialize
+      @mapping = {}
+    end
+
+    def objenc_internal(x)
+      x.map{|k,v| "#{keyenc(k)}=#{valenc(v)}" }.join(' ')
+    end
+
+    def keyenc(k)
+      case k
+      when String, Symbol then strenc(k)
+      else
+        strenc(k.inspect)
+      end
+    end
+
+    def valenc(x)
+      case x
+      when Hash           then objenc(x)
+      when Array          then arrenc(x)
+      when String, Symbol then strenc(x)
+      when Numeric        then numenc(x)
+      when Time, Date     then timeenc(x)
+      when true           then "#t"
+      when false          then "#f"
+      when nil            then "nil"
+      else
+        litenc(x)
+      end
+    end
+
+    def objenc(x)
+      '{' + objenc_internal(x) + '}'
+    end
+
+    def arrenc(a)
+      # num + unit. Eg: 3ms
+      if a.size == 2 && a.first.kind_of?(Numeric) && is_literal?(a.last.to_s)
+        numenc(a.first) + strenc(a.last)
+      else
+        '[' + a.map{|x| valenc(x)}.join(' ') + ']'
+      end
+    end
+
+    # TODO: Single-quote espace if possible
+    def strenc(s)
+      s = s.to_s
+      s = s.inspect unless is_literal?(s)
+      s
+    end
+
+    def numenc(n)
+      #case n
+      # when Float
+      #   "%.3f" % n
+      #else
+        n.to_s
+      #end
+    end
+
+    def litenc(x)
+      klass = (x.class.ancestors & mapping.keys).first
+      if klass
+        mapping[klass].call(x)
+      else
+        strenc(x.inspect)
+      end
+    rescue
+      klass = (class << x; self; end).ancestors.first
+      strenc("#<#{klass}:0x#{x.__id__.to_s(16)}>")
+    end
+
+    def timeenc(t)
+      t.iso8601
+    end
+
+    def is_literal?(s)
+      !s.index(/[\s'"]/)
+    end
+
+  end
+
+  require 'securerandom'
+  module UniqueIDs
+    # A small utility to generate unique IDs that are as short as possible.
+    #
+    # It's useful to link contextes together
+    #
+    # See http://preshing.com/20110504/hash-collision-probabilities
+    def id(collision_chance=1.0/10e9, over_x_messages=10e3)
+      # Assuming that the distribution is perfectly random
+      # how many bits do we need so that the chance of collision over_x_messages
+      # is lower thant collision_chance ? 
+      number_of_possible_numbers = (over_x_messages ** 2) / (2 * collision_chance)
+      num_bytes = (Math.log2(number_of_possible_numbers) / 8).ceil
+      SecureRandom.urlsafe_base64(num_bytes)
+    end
+  end
+  extend UniqueIDs
+end

data/lib/lines/active_record.rb

@@ -0,0 +1,36 @@
+require 'active_record'
+require 'lines'
+
+module Lines
+  class ActiveRecordSubscriber < ActiveSupport::LogSubscriber
+    def sql(event)
+      payload = event.payload
+
+      return if payload[:name] == "SCHEMA"
+
+      args = {}
+
+      args[:name] = payload[:name] if payload[:name]
+      args[:sql] = payload[:sql].squeeze(' ')
+
+      if payload[:binds] && payload[:binds].any?
+        args[:binds] = payload[:binds].inject({}) do |hash,(col, v)|
+          hash[col.name] = v
+          hash
+        end
+      end
+
+      args[:elapsed] = [event.duration, 's']
+
+      Lines.log(args)
+    end
+
+    def identity(event)
+      Lines.log(name: event.payload[:name], line: event.payload[:line])
+    end
+
+    def logger; true; end
+  end
+end
+
+Lines::ActiveRecordSubscriber.attach_to :active_record

data/lib/lines/loader.rb

@@ -0,0 +1,166 @@
+begin
+  require 'parslet'
+rescue LoadError
+  warn "lines/loader depends on parslet"
+  raise
+end
+
+# http://zerowidth.com/2013/02/24/parsing-toml-in-ruby-with-parslet.html
+module Lines
+  module Error; end
+  module ParseError; include Error; end
+  module Loader; extend self
+    def load(s)
+      parser = Parser.new
+      transformer = Transformer.new
+
+      tree = parser.parse(s)
+      #puts; p tree; puts
+      transformer.apply(tree)
+    rescue Parslet::ParseFailed => ex
+      # Mark as being part of the Lines library
+      ex.extend ParseError
+      raise
+    end
+  end
+
+  # Mostly copied over from the JSON example:
+  #   https://github.com/kschiess/parslet/blob/master/example/json.rb
+  #
+  # TODO:
+  #   ISO8601 dates
+  class Parser < Parslet::Parser
+
+    rule(:spaces) { match(' ').repeat(1) }
+    rule(:spaces?) { spaces.maybe }
+
+    rule(:digit) { match['0-9'] }
+
+    rule(:number) {
+      (
+        str('-').maybe >> (
+          str('0') | (match['1-9'] >> digit.repeat)
+        ) >> (
+          str('.') >> digit.repeat(1)
+        ).maybe >> (
+          match('[eE]') >> (str('+') | str('-')).maybe >> digit.repeat(1)
+        ).maybe
+      ).as(:number)
+    }
+
+    rule(:time) {
+      digit.repeat(4) >> str('-') >>
+      digit.repeat(2) >> str('-') >>
+      digit.repeat(2) >> str('T') >>
+      digit.repeat(2) >> str(':') >>
+      digit.repeat(2) >> str(':') >>
+      digit.repeat(2) >> str('Z')
+    }
+
+    rule(:singlequoted_string) {
+      str("'") >> (
+        str('\\') >> any | str("'").absent? >> any
+      ).repeat.as(:string) >> str("'")
+    }
+
+    rule(:doublequoted_string) {
+      str('"') >> (
+        str('\\') >> any | str('"').absent? >> any
+      ).repeat.as(:string) >> str('"')
+    }
+
+    rule(:simple_string) {
+      match['a-zA-Z_\-:'].repeat.as(:string)
+    }
+
+    rule(:string) {
+      singlequoted_string | doublequoted_string | simple_string
+    }
+
+    rule(:array) {
+      str('[') >> spaces? >>
+      (value >> (spaces >> value).repeat).maybe.as(:array) >>
+      spaces? >> str(']')
+    }
+
+    rule(:object) {
+      str('{') >> spaces? >>
+      (entry >> (spaces >> entry).repeat).maybe.as(:object) >>
+      spaces? >> str('}')
+    }
+
+    rule(:key) {
+      match['a-zA-Z0-9_'].repeat
+    }
+
+    rule(:value) {
+      str('#t').as(:true) | str('#f').as(:false) |
+      str('nil').as(:nil) |
+      object | array |
+      number | time |
+      string
+    }
+
+    rule(:entry) {
+      (
+         key.as(:key) >>
+         str('=') >>
+         value.as(:val)
+      ).as(:entry)
+    }
+
+    rule(:top) { spaces? >> (entry >> (spaces >> entry).repeat).maybe.as(:object) >> spaces? }
+    #rule(:top) { (digit >> digit).as(:digit) }
+    #rule(:top) { time }
+
+    root(:top)
+  end
+
+  class Transformer < Parslet::Transform
+
+    class Entry < Struct.new(:key, :val); end
+
+    rule(array: subtree(:ar)) {
+      case ar
+      when nil
+        []
+      when Array
+        ar
+      else
+        [ar]
+      end
+    }
+    rule(object: subtree(:ob)) {
+      case ob
+      when nil
+        []
+      when Array
+        ob
+      else
+        [ob]
+      end.inject({}) { |h, e|
+        h[e[:entry][:key].to_s] = e[:entry][:val]; h
+      }
+    }
+
+    # rule(entry: { key: simple(:ke), val: simple(:va) }) {
+    #   Entry.new(ke.to_s, va)
+    # }
+
+    rule(time: { year: simple(:ye), month: simple(:mo), day: simple(:da), hour: simple(:ho), minute: simple(:min), second: simple(:sec)}) {
+      Time.new(ye.to_i, mo.to_i, da.to_i, ho.to_i, min.to_i, sec.to_i, "+00:00")
+    }
+
+    rule(string: simple(:st)) {
+      st.to_s
+    }
+
+    rule(number: simple(:nb)) {
+      nb.match(/[eE\.]/) ? Float(nb) : Integer(nb)
+    }
+
+    rule(nil: simple(:ni)) { nil }
+    rule(true: simple(:tr)) { true }
+    rule(false: simple(:fa)) { false }
+  end
+end

data/lib/lines/logger.rb

@@ -0,0 +1,61 @@
+module Lines
+  # Backward-compatible logger
+  # http://ruby-doc.org/stdlib-2.0/libdoc/logger/rdoc/Logger.html#method-i-log
+  class Logger
+    LEVELS = {
+      0 => :debug,
+      1 => :info,
+      2 => :warn,
+      3 => :error,
+      4 => :fatal,
+      5 => :unknown,
+    }
+
+    def initialize(line)
+      @line = line
+    end
+
+    def log(severity, message = nil, progname = nil, &block)
+      pri = LEVELS[severity] || severity
+      if block_given?
+        progname = message
+        message = yield.to_s rescue $!.to_s
+      end
+
+      data = { pri: pri }
+      data[:app] = progname if progname
+      data[:msg] = message if message
+
+      @line.log(data)
+    end
+
+    LEVELS.values.each do |level|
+      define_method(level) do |message=nil, &block|
+        log(level, message, &block)
+      end
+    end
+
+    alias << info
+    alias unknown info
+
+    def noop(*a); true end
+    %w[add
+      clone
+      datetime_format
+      datetime_format=
+      debug?
+      info?
+      error?
+      fatal?
+      warn?
+      level
+      level=
+      progname
+      progname=
+      sev_threshold
+      sev_threshold=
+    ].each do |op|
+      alias_method(op, :noop)
+    end
+  end
+end

data/lib/lines/rack_logger.rb

@@ -0,0 +1,38 @@
+require 'rack/commonlogger'
+require 'lines'
+
+module Lines
+  class RackLogger < Rack::CommonLogger
+    # In development mode the common logger is always inserted
+    def self.silence_common_logger!
+      Rack::CommonLogger.module_eval("def call(env); @app.call(env); end")
+    end
+
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      began_at = Time.now
+      status, header, body = @app.call(env)
+      header = Utils::HeaderHash.new(header)
+      body = BodyProxy.new(body) { log(env, status, header, began_at) }
+      [status, header, body]
+    end
+
+    protected
+
+    def log(env, status, header, began_at)
+      Lines.log(
+        remote_addr: env['HTTP_X_FORWARDED_FOR'] || env["REMOTE_ADDR"],
+        remote_user: env['REMOTE_USER'] || '',
+        method: env['REQUEST_METHOD'],
+        path: env['PATH_INFO'],
+        query:  env["QUERY_STRING"],
+        status: status.to_s[0..3],
+        length: extract_content_length(header),
+        elapsed: [Time.now - began_at, 's'],
+      )
+    end
+  end
+end

data/lib/lines/version.rb

@@ -0,0 +1,3 @@
+module Lines
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,51 @@
+--- !ruby/object:Gem::Specification
+name: lines
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Jonas Pfenniger
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-03 00:00:00.000000000 Z
+dependencies: []
+description: Lines is a cross-language logging format
+email: jonas@pfenniger.name
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/lines/active_record.rb
+- lib/lines/loader.rb
+- lib/lines/logger.rb
+- lib/lines/rack_logger.rb
+- lib/lines/version.rb
+- lib/lines.rb
+homepage: https://github.com/zimbatm/lines
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Logging revisited
+test_files: []