lines 0.2.0 → 0.9.1

data/lib/lines/common.rb

@@ -0,0 +1,37 @@
+# -*- encoding : utf-8 -*-
+
+module Lines
+  module Error
+    # Used to mark non-lines errors as being part of the library. This lets
+    # a library user `rescue Lines::Error => ex` and catch all exceptions
+    # comming from the lines library.
+    def self.tag(obj)
+      obj.extend Error
+    end
+  end
+
+  class ParseError < StandardError; include Error; end
+  #class LogicError < RuntimeError; include Error; end
+
+  LIT_TRUE  = '#t'
+  LIT_FALSE = '#f'
+  LIT_NIL   = 'nil'
+
+  SPACE         = ' '
+  EQUAL         = '='
+  OPEN_BRACE    = '{'
+  SHUT_BRACE    = '}'
+  OPEN_BRACKET  = '['
+  SHUT_BRACKET  = ']'
+  SINGLE_QUOTE  = "'"
+  DOUBLE_QUOTE  = '"'
+  DOT_DOT_DOT   = '...'
+
+  BACKSLASH             = '\\'
+  ESCAPED_SINGLE_QUOTE  = "\\'"
+  ESCAPED_DOUBLE_QUOTE  = '\"'
+
+  NUM_MATCH             = /-?(?:0|[1-9])\d*(?:\.\d+)?(?:[eE][+-]\d+)?/
+  ISO8601_ZULU_CAPTURE  = /^(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)Z$/
+  NUM_CAPTURE           = /^(#{NUM_MATCH})$/
+end

data/lib/lines/generator.rb

@@ -0,0 +1,168 @@
+# -*- encoding : utf-8 -*-
+
+require 'time'
+
+require 'lines/common'
+
+module Lines
+  # Some opinions here as well on the format:
+  #
+  # We really want to never fail at dumping because you know, they're logs.
+  # It's better to get a slightly less readable log that no logs at all.
+  #
+  # We're trying to be helpful for humans. It means that if possible we want
+  # to make things shorter and more readable. It also means that ideally
+  # we would like the parsing to be isomorphic but approximations are alright.
+  # For example a symbol might become a string.
+  #
+  # Basically, values are either composite (dictionaries and arrays), quoted
+  # strings or litterals. Litterals are strings that can be parsed to
+  # something else depending if the language supports it or not.
+  # Litterals never contain white-spaces or other weird (very precise !) characters.
+  #
+  # the true litteral is written as "#t"
+  # the false litteral is written as "#f"
+  # the nil / null litteral is written as "nil"
+  #
+  # dictionary keys are always strings or litterals.
+  #
+  # Pleaaase, keep units with numbers. And we provide a way for this:
+  # a tuple of (number, litteral) can be concatenated. Eg: (3, 'ms') => 3:ms
+  # alternatively if your language supports a time range it could be serialized
+  # to the same value (and parsed back as well).
+  #
+  # if we don't know how to serialize something we provide a language-specific
+  # string of it and encode is at such.
+  #
+  # The output ought to use the UTF-8 encoding.
+  #
+  # This dumper has been inspired by the OkJSON gem (both formats look alike
+  # after all).
+  module Generator; extend self
+    SINGLE_QUOTE_MATCH  = /'/
+    STRING_ESCAPE_MATCH = /[\s"=:{}\[\]]/
+
+    # max_nesting::
+    #   After a certain depth, arrays are replaced with [...] and objects with
+    #   {...}. Default is 4
+    # max_bytesize::
+    #   After a certain lenght the root object is interrupted by the ...
+    #   notation. Default is 4096.
+    def generate(obj, opts={}) #=> String
+      max_nesting = opts[:max_nesting] || 4
+      max_bytesize = opts[:max_bytesize] || 4096
+
+      depth = max_nesting - 1
+      bytesize = 0
+
+      line = obj.inject([]) do |acc, (k, v)|
+        if bytesize + (acc.size - 1) > max_bytesize
+          break acc
+        end
+        str = "#{keyenc(k)}=#{valenc(v, depth)}"
+        bytesize += str.bytesize
+        acc.push(str)
+        acc
+      end
+      if bytesize + (line.size - 1) > max_bytesize
+        if bytesize + (line.size - 1) - (line[-1].bytesize - 3) > max_bytesize
+          line.pop
+        end
+        line[-1] = DOT_DOT_DOT
+      end
+      line.join(SPACE)
+    end
+
+    protected
+
+    def valenc(x, depth)
+      case x
+      when Hash           then objenc(x, depth)
+      when Array          then arrenc(x, depth)
+      when String, Symbol then strenc(x)
+      when Numeric        then numenc(x)
+      when Time           then timeenc(x)
+      when true           then LIT_TRUE
+      when false          then LIT_FALSE
+      when nil            then LIT_NIL
+      else
+        litenc(x)
+      end
+    end
+
+    def objenc_internal(x, depth)
+      depth -= 1
+      if depth < 0
+        DOT_DOT_DOT
+      else
+        x.map{|k,v| "#{keyenc(k)}=#{valenc(v, depth)}" }.join(SPACE)
+      end
+    end
+
+    def objenc(x, depth)
+      OPEN_BRACE + objenc_internal(x, depth) + SHUT_BRACE
+    end
+
+    def arrenc_internal(a, depth)
+      depth -= 1
+      if depth < 0
+        DOT_DOT_DOT
+      else
+        a.map{|x| valenc(x, depth)}.join(SPACE)
+      end
+    end
+
+    def arrenc(a, depth)
+      OPEN_BRACKET + arrenc_internal(a, depth) + SHUT_BRACKET
+    end
+
+    def keyenc(s)
+      s = s.to_s
+      # Poor-man's escaping
+      case s
+      when SINGLE_QUOTE_MATCH
+        s.inspect
+      when STRING_ESCAPE_MATCH
+        SINGLE_QUOTE +
+          s.inspect[1..-2].gsub(ESCAPED_DOUBLE_QUOTE, DOUBLE_QUOTE) +
+        SINGLE_QUOTE
+      else
+        s
+      end
+    end
+
+    def strenc(s)
+      s = s.to_s
+      # Poor-man's escaping
+      case s
+      when SINGLE_QUOTE_MATCH
+        s.inspect
+      when STRING_ESCAPE_MATCH, NUM_CAPTURE, LIT_TRUE, LIT_FALSE, LIT_NIL, DOT_DOT_DOT
+        SINGLE_QUOTE +
+          s.inspect[1..-2].gsub(ESCAPED_DOUBLE_QUOTE, DOUBLE_QUOTE) +
+        SINGLE_QUOTE
+      else
+        s
+      end
+    end
+
+    def numenc(n)
+      #case n
+      # when Float
+      #   "%.3f" % n
+      #else
+        n.to_s
+      #end
+    end
+
+    def litenc(x)
+      strenc x.inspect
+    rescue
+      strenc (class << x; self; end).ancestors.first.inspect
+    end
+
+    def timeenc(t)
+      t.utc.iso8601
+    end
+  end
+end

data/lib/lines/parser.rb

@@ -0,0 +1,182 @@
+# -*- encoding : utf-8 -*-
+
+require 'strscan'
+
+require 'lines/common'
+
+module Lines
+  class Parser
+    DOT_DOT_DOT_MATCH     = /\.\.\./
+
+    LITERAL_MATCH         = /[^=\s}\]]+/
+    SINGLE_QUOTE_MATCH    = /(?:\\.|[^'])*/
+    DOUBLE_QUOTE_MATCH    = /(?:\\.|[^"])*/
+
+    def self.parse(string, opts={})
+      new(string, opts).parse
+    end
+
+    def initialize(string, opts)
+      @s = StringScanner.new(string)
+      @opts = opts
+    end
+
+    def parse
+      inner_obj
+    end
+
+    protected
+
+    def accept(char)
+      if @s.peek(1) == char
+        @s.pos += 1
+        return true
+      end
+      false
+    end
+
+    def skip(num)
+      @s.pos += num
+    end
+
+    def expect(char)
+      if !accept(char)
+        fail "Expected '#{char}' but got '#{@s.peek(1)}'"
+      end
+    end
+
+    def fail(msg)
+      raise ParseError, "At #{@s}, #{msg}"
+    end
+
+    def dbg(*x)
+      #p [@s] + x
+    end
+
+    # Structures
+
+
+    def inner_obj
+      dbg :inner_obj
+      # Shortcut for the '...' max_depth notation
+      if @s.scan(DOT_DOT_DOT_MATCH)
+        return {DOT_DOT_DOT => ''}
+      end
+
+      return {} if @s.eos? || @s.peek(1) == SHUT_BRACE
+
+      # First pair
+      k = key()
+      expect EQUAL
+      obj = {
+        k => value()
+      }
+
+      while accept(SPACE) and !@s.eos?
+        k = key()
+        expect EQUAL
+        obj[k] = value()
+      end
+
+      obj
+    end
+
+    def key
+      dbg :key
+
+      ret = case @s.peek(1)
+      when SINGLE_QUOTE
+        single_quoted_string
+      when DOUBLE_QUOTE
+        double_quoted_string
+      else
+        literal(false)
+      end
+      @opts[:symbolize_names] ? ret.to_sym : ret
+    end
+
+    def single_quoted_string
+      dbg :single_quoted_string
+
+      expect SINGLE_QUOTE
+      str = @s.scan(SINGLE_QUOTE_MATCH).
+        gsub(ESCAPED_SINGLE_QUOTE, SINGLE_QUOTE)
+      expect SINGLE_QUOTE
+      str
+    end
+
+    def double_quoted_string
+      dbg :double_quoted_string
+
+      expect DOUBLE_QUOTE
+      str = @s.scan(DOUBLE_QUOTE_MATCH).
+        gsub(ESCAPED_DOUBLE_QUOTE, DOUBLE_QUOTE)
+      expect DOUBLE_QUOTE
+      str
+    end
+
+    def literal(sub_parse)
+      dbg :literal, sub_parse
+
+      literal = @s.scan LITERAL_MATCH
+
+      return "" unless literal
+      
+      return literal unless sub_parse
+
+      case literal
+      when LIT_NIL
+        nil
+      when LIT_TRUE
+        true
+      when LIT_FALSE
+        false
+      when ISO8601_ZULU_CAPTURE
+        Time.new($1.to_i, $2.to_i, $3.to_i, $4.to_i, $5.to_i, $6.to_i, '+00:00').utc
+      when NUM_CAPTURE
+        literal.index('.') ? Float(literal) : Integer(literal)
+      else
+        literal
+      end
+    end
+
+    def value
+      dbg :value
+
+      case @s.peek(1)
+      when OPEN_BRACKET
+        list
+      when OPEN_BRACE
+        object
+      when DOUBLE_QUOTE
+        double_quoted_string
+      when SINGLE_QUOTE
+        single_quoted_string
+      else
+        literal(:sub_parse)
+      end
+    end
+
+    def list
+      dbg :list
+
+      list = []
+      expect(OPEN_BRACKET)
+      list.push value()
+      while accept(SPACE)
+        list.push value()
+      end
+      expect(SHUT_BRACKET)
+      list
+    end
+
+    def object
+      dbg :object
+
+      expect(OPEN_BRACE)
+      obj = inner_obj
+      expect(SHUT_BRACE)
+      obj
+    end
+  end
+end

data/lib/lines/version.rb

@@ -1,3 +1,3 @@
 module Lines
-  VERSION = "0.2.0"
+  VERSION = "0.9.1"
 end

data/lines.gemspec

@@ -1,23 +1,25 @@
 # coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'lines/version'
+require File.expand_path('../lib/lines/version', __FILE__)
 
 Gem::Specification.new do |spec|
   spec.name          = "lines"
   spec.version       = Lines::VERSION
-  spec.authors       = ["Jonas Pfenniger"]
-  spec.email         = ["jonas@pfenniger.name"]
-  spec.description   = %q{structured logs for humans}
-  spec.summary       = %q{Lines is an opinionated structured logging library}
+  spec.authors       = ["zimbatm"]
+  spec.email         = ["zimbatm@zimbatm.com"]
+  spec.summary       = %q{Lines is an opinionated structured log format}
+  spec.description   = <<DESC
+A log format that's readable by humans and easily parseable by computers.
+DESC
   spec.homepage      = 'https://github.com/zimbatm/lines-ruby'
   spec.license       = "MIT"
 
-  spec.files         = `git ls-files`.split($/)
+  spec.files         = `git ls-files .`.split($/)
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.add_development_dependency "benchmark-ips"
   spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
 end

data/spec/lines_generator_bench.rb

@@ -0,0 +1,45 @@
+$:.unshift File.expand_path('../../lib', __FILE__)
+
+require 'benchmark/ips'
+require 'time'
+
+$message = {
+  "at" => Time.now.utc.iso8601,
+  "pid" => Process.pid,
+  "app" => File.basename($0),
+  "pri" => "info",
+  "msg" => "This is my message",
+  "user" => {"t" => true, "f" => false, "n" => nil},
+  "elapsed" => [55.67, 'ms'],
+}
+
+formatters = [
+  ['lines', "Lines.dump($message)"],
+
+  ['json/pure', "JSON.dump($message)"],
+  ['oj', "Oj.dump($message)"],
+  ['yajl', "Yajl.dump($message)"],
+  
+  ['msgpack', "MessagePack.dump($message)"],
+  ['bson', "$message.to_bson"],
+  ['tnetstring', "TNetstring.dump($message)"],
+]
+
+puts "%-12s %-5s %s" % ['format', 'size', 'output']
+puts "-" * 25
+
+Benchmark.ips do |x|
+  x.compare!
+  formatters.each do |(feature, action)|
+    begin
+      require feature
+
+      data = eval action
+      puts "%-12s %-5d %s" % [feature, data.size, data]
+
+      x.report feature, action
+    rescue LoadError
+      puts "%-12s could not be loaded" % [feature]
+    end
+  end
+end