lines 0.1.27 → 0.2.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OTM0ODk2MzlkMTQ3YmJhMTc2MmI4MGMzNzYzNTQyMTFjNmQ5OTYyZQ==
+    ZGM0YmFkODI1ZDgwOTA1NTQ1Y2JmZjM5ZDE2ZjIxMzAxM2JhNWZhZg==
   data.tar.gz: !binary |-
-    OTA4YzE1ZTJkMjQwZDM0NmRiOGVlMDY1MzFkMmIxYmJhMzYxNzlmZA==
+    YzY3M2FhMGEyNmExZGEyODZiMDIyMDgzOWQ4YTQ4NThiMzM2NDI3NQ==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    YWZlYzZmYzdhNjBjOTIwNGU2MjYyMjFmMjNkNGNjYzE1NmI2NTk2NzJlOGYx
-    MGRhNmU2ZjYxNWY3MGJlMWExMjVjYzEwMTBiZGU0MGFlMWIyNzEyNjE0NjQ4
-    YzA0Y2QwZTc4NmI4N2E3YmI4OGQwZDY4Y2E4OTI5NmI2N2FmNzY=
+    NTI3YzJhY2M5YjMzZDE2NWI3NjdjYmE0MDZjOWU0MDg1MjY0ZTY2NmYzYThh
+    ZjkxNTQ4ZjZjMzQ0OGRiMTZmNTc5ZDhmYjI3ODIwMDg4MDg0NTFmYWE4OWU3
+    MmUyYzgxOTU0MjQ0N2NhNDFmNThkZmU3Y2NjOWMyZWUxMWIwMmU=
   data.tar.gz: !binary |-
-    NWFkZGFkNjY5OWE4NmViZWEzOWNhNzllNjZiZTg1NzI2YmFjNzRmOTYyNGE0
-    OTM4ZDA0YTc1ZmM5MzdmYzdlY2I0YzMwOTVhNDE0YTBjYjk5YzM0MjVhNGEw
-    NTUxOWIyOTkzOGExNDliZjY1ZTI0NTUyYzVlYjlkNTY3ZWE5Yzg=
+    Njc3MjM5ZjJmMmY0MWE4OWM1ZmFlYmUyNzdlMmQ0OWFmZTVmZTllZjc0YWRk
+    ZDczMjM4YmEwYTMwYTU1OTU5Y2E1OWIwNjAzMGNiYTc1NTdiNGJhNWUzZjE5
+    ZjhiNDlhYWU1ZGM4MTBmZTQ3YmY4ZmQyNjMxOWU5NDMyZjZiMWU=

data/CHANGELOG.md

@@ -1,4 +1,21 @@
 
+0.2.0 / 2013-07-15 
+==================
+
+ * Use benchmark-ips for benchs
+ * Fixes serialization of Date objects
+ * Lines now outputs to $stderr by default.
+ * Lines.use resets the global context.
+ * Improved the doc
+ * Lines.log now returns nil instead of the logged object.
+ * Support parsing lines that end with \r\n or spaces
+ * Add Lines.load and Lines.dump for JSON-like functionality
+ * Introduced a hand-written parser that performs 200x faster
+ * Differentiate units with a : sign to ensure their parsability
+ * Escape strings that contain an equal sign
+ * Change the default max_depth from 3 to 4
+ * Make sure ActiveRecord's log subscriber is loaded
+
 0.1.27 / 2013-07-10 
 ===================
 

data/Gemfile

@@ -1,10 +1,10 @@
 source 'http://rubygems.org'
 gemspec
 group :development do
-  gem 'parslet'
   gem 'rdoc'
 end
 group :test do
-  gem 'rspec'
+  gem 'benchmark-ips'
   gem 'rake'
+  gem 'rspec'
 end

data/README.md

@@ -6,10 +6,85 @@ Status](https://travis-ci.org/zimbatm/lines-ruby.png)](https://travis-ci.org/zim
 An oppinionated logging library that implement the
 [lines](https://github.com/zimbatm/lines) format.
 
-Status: work in progress
+* Log everything in development AND production.
+* Logs should be easy to read, grep and parse.
+* Logging something should never fail.
+* Let the system handle the storage. Write to syslog or STDERR.
+* No log levels necessary. Just log whatever you want.
 
-See also
+STATUS: WORK IN PROGRESS
+========================
+
+Doc is still scarce so it's quite hard to get started. I think reading the
+lib/lines.rb should give a good idea of the capabilities.
+
+Lines.id is a unique ID generator that seems quite handy but I'm not sure if
+it should be part of the lib or not.
+
+It would be nice to expose a method that resolves a context into a hash. It's
+useful to share the context with other tools like an error reporter. Btw,
+Sentry/Raven is great.
+
+There is a parser in the lib but no credible direct consumption path.
+
+Quick intro
+-----------
+
+```ruby
+require 'lines'
+
+# Setups the outputs. IO and Syslog are supported.
+Lines.use($stdout, Syslog)
+
+# All lines will be prefixed by the global context
+Lines.global['at'] = proc{ Time.now }
+
+# First example
+Lines.log(foo: 'bar') # logs: at=2013-07-14T14:19:28Z foo=bar
+
+# If not a hash, the argument is transformed. A second argument is accepted as
+# a hash
+Lines.log("Hey", count: 3) # logs: at=2013-07-14T14:19:28Z msg=Hey count=3
+
+# You can also keep a context
+class MyClass < ActiveRecord::Base
+  attr_reader :lines
+  def initialize
+    @lines = Lines.context(my_class_id: self.id)
+  end
+
+  def do_something
+    lines.log("Something happened")
+    # logs: at=2013-07-14T14:19:28Z msg='Something happeend' my_class_id: 2324
+  end
+end
+```
+
+Features
 --------
 
+* Simple to use
+* Thread safe (if the IO#write is)
+* Designed to not raise exceptions (unless it's an IO issue)
+* Lines.logger is a backward-compatible Logger in case you want to retrofit
+* require "lines/active_record" for sane ActiveRecord logs
+* "lines/rack_logger" is a logging middleware for Rack
+* Lines.load and Lines.dump to parse and generate 'lines'
+
+There's also a fork of lograge that you can use with Rails. See
+https://github.com/zimbatm/lograge/tree/lines-output
+
+Known issues
+------------
+
+Syslog seems to truncate lines longer than 2056 chars and Lines makes if very
+easy to put too much data.
+
+Lines logging speed is reasonable but it could be faster. It writes at around
+5000 lines per second to Syslog on my machine.
+
+Inspired by
+-----------
+
  * Scrolls : https://github.com/asenchi/scrolls
  * Lograge : https://github.com/roidrage/lograge

data/lib/lines.rb

@@ -1,19 +1,17 @@
 require 'date'
 require 'time'
-require 'forwardable'
 
-# Lines is an opinionated structured log format and a library
-# inspired by Slogger.
+# Lines is an opinionated structured log format and a library.
 #
-# Don't use log levels. They limit the reasoning of the developer.
 # Log everything in development AND production.
 # Logs should be easy to read, grep and parse.
 # Logging something should never fail.
-# Use syslog.
+# Let the system handle the storage. Write to syslog or STDERR.
+# No log levels necessary. Just log whatever you want.
 #
 # Example:
 #
-#     log(msg: "Oops !")
+#     log("Oops !", foo: {}, g: [])
 #     #outputs:
 #     # at=2013-03-07T09:21:39+00:00 pid=3242 app=some-process msg="Oops !" foo={} g=[]
 #
@@ -25,40 +23,57 @@ require 'forwardable'
 #     ctx = Lines.context(encoding_id: Log.id)
 #     ctx.log({})
 #
-#     Lines.context(:foo => :bar) do |l|
-#       l.log(:sadfasdf => 3)
+#     Lines.context(foo: 'bar') do |l|
+#       l.log(items_count: 3)
 #     end
 module Lines
-  # New lines in Lines
-  NL = "\n".freeze
+  class << self
+    attr_reader :global
+    attr_writer :loader, :dumper
 
-  @global = {}
-  @outputters = []
+    # Parsing object. Responds to #load(string)
+    def loader
+      @loader ||= (
+        require 'lines/loader'
+        Loader
+      )
+    end
 
-  class << self
+    # Serializing object. Responds to #dump(hash)
     def dumper; @dumper ||= Dumper.new end
-    attr_reader :global
-    attr_reader :outputters
 
-    # Used to select what output the lines will be put on.
+    # Returns a backward-compatibile Logger
+    def logger
+      @logger ||= (
+        require 'lines/logger'
+        Logger.new(self)
+      )
+    end
+
+    # Used to configure lines.
     #
     # outputs - allows any kind of IO or Syslog
     #
     # Usage:
     #
-    #     Lines.use(Syslog, $stderr)
+    #     Lines.use(Syslog, $stderr, at: proc{ Time.now })
     def use(*outputs)
-      outputters.replace(outputs.flatten.map{|o| to_outputter o})
+      if outputs.last.kind_of?(Hash)
+        @global = outputs.pop
+      else
+        @global = {}
+      end
+      @outputters = outputs.flatten.map{|o| to_outputter o}
     end
 
     # The main function. Used to record objects in the logs as lines.
     #
-    # obj - a ruby hash
-    # args -
+    # obj - a ruby hash. coerced to +{"msg"=>obj}+ otherwise
+    # args - complementary values to put in the line
     def log(obj, args={})
       obj = prepare_obj(obj, args)
-      outputters.each{|out| out.output(dumper, obj) }
-      obj
+      @outputters.each{|out| out.output(dumper, obj) }
+      nil
     end
 
     # Add data to the logs
@@ -72,19 +87,21 @@ module Lines
       new_context
     end
 
-    # Returns a backward-compatibile logger
-    def logger
-      @logger ||= (
-        require 'lines/logger'
-        Logger.new(self)
-      )
-    end
-
     def ensure_hash!(obj) # :nodoc:
       return {} unless obj
       return obj if obj.kind_of?(Hash)
       return obj.to_h if obj.respond_to?(:to_h)
-      obj = {msg: obj}
+      {msg: obj}
+    end
+
+    # Parses a lines-formatted string
+    def load(string)
+      loader.load(string)
+    end
+
+    # Generates a lines-formatted string from the given object
+    def dump(obj)
+      dumper.dump ensure_hash!(obj)
     end
 
     protected
@@ -118,7 +135,7 @@ module Lines
     end
   end
 
-  # Wrapper object that holds a given context. Emitted by Lines.with
+  # Wrapper object that holds a given context. Emitted by Lines.context
   class Context
     attr_reader :data
 
@@ -126,12 +143,16 @@ module Lines
       @data = data
     end
 
+    # Works like the Lines.log method.
     def log(obj, args={})
       Lines.log obj, Lines.ensure_hash!(args).merge(data)
     end
   end
 
+  # Handles output to any kind of IO
   class StreamOutputter
+    NL = "\n".freeze
+
     # stream must accept a #write(str) message
     def initialize(stream = $stderr)
       @stream = stream
@@ -141,28 +162,25 @@ module Lines
 
     def output(dumper, obj)
       str = dumper.dump(obj) + NL
-      stream.write str
+      @stream.write str
     end
-
-    protected
-
-    attr_reader :stream
   end
 
   require 'syslog'
+  # Handles output to syslog
   class SyslogOutputter
     PRI2SYSLOG = {
-      'debug'    => Syslog::LOG_DEBUG,
-      'info'     => Syslog::LOG_INFO,
-      'warn'     => Syslog::LOG_WARNING,
-      'warning'  => Syslog::LOG_WARNING,
-      'err'      => Syslog::LOG_ERR,
-      'error'    => Syslog::LOG_ERR,
-      'crit'     => Syslog::LOG_CRIT,
-      'critical' => Syslog::LOG_CRIT,
-    }
-
-    def initialize(syslog = Syslog)
+      'debug'    => ::Syslog::LOG_DEBUG,
+      'info'     => ::Syslog::LOG_INFO,
+      'warn'     => ::Syslog::LOG_WARNING,
+      'warning'  => ::Syslog::LOG_WARNING,
+      'err'      => ::Syslog::LOG_ERR,
+      'error'    => ::Syslog::LOG_ERR,
+      'crit'     => ::Syslog::LOG_CRIT,
+      'critical' => ::Syslog::LOG_CRIT,
+    }.freeze
+
+    def initialize(syslog = ::Syslog)
       @syslog = syslog
     end
 
@@ -175,9 +193,8 @@ module Lines
       obj.delete(:app) # And again
 
       level = extract_pri(obj)
-      str = dumper.dump(obj)
 
-      @syslog.log(level, "%s", str)
+      @syslog.log(level, "%s", dumper.dump(obj))
     end
 
     protected
@@ -186,13 +203,13 @@ module Lines
       return if @syslog.opened?
       app_name ||= File.basename($0)
       @syslog.open(app_name,
-                  Syslog::LOG_PID | Syslog::LOG_CONS | Syslog::LOG_NDELAY,
-                  Syslog::LOG_USER)
+                  ::Syslog::LOG_PID | ::Syslog::LOG_CONS | ::Syslog::LOG_NDELAY,
+                  ::Syslog::LOG_USER)
     end
 
     def extract_pri(h)
       pri = h.delete(:pri).to_s.downcase
-      PRI2SYSLOG[pri] || Syslog::LOG_INFO
+      PRI2SYSLOG[pri] || ::Syslog::LOG_INFO
     end
   end
 
@@ -230,15 +247,41 @@ module Lines
   # This dumper has been inspired by the OkJSON gem (both formats look alike
   # after all).
   class Dumper
+    SPACE = ' '
+    LIT_TRUE = '#t'
+    LIT_FALSE = '#f'
+    LIT_NIL = 'nil'
+    OPEN_BRACE = '{'
+    SHUT_BRACE = '}'
+    OPEN_BRACKET = '['
+    SHUT_BRACKET = ']'
+    SINGLE_QUOTE = "'"
+    DOUBLE_QUOTE = '"'
+
+    constants.each(&:freeze)
+
     def dump(obj) #=> String
       objenc_internal(obj)
     end
 
     # Used to introduce new ruby litterals.
+    #
+    # Usage:
+    #
+    #     Point = Struct.new(:x, :y)
+    #     Lines.dumper.map(Point) do |p|
+    #       "#{p.x}x#{p.y}"
+    #     end
+    #
+    #     Lines.log msg: Point.new(3, 5)
+    #     # logs: msg=3x5
+    #
     def map(klass, &rule)
       @mapping[klass] = rule
     end
 
+    # After a certain depth, arrays are replaced with [...] and objects with
+    # {...}. Default is 4.
     attr_accessor :max_depth
 
     protected
@@ -247,7 +290,7 @@ module Lines
 
     def initialize
       @mapping = {}
-      @max_depth = 3
+      @max_depth = 4
     end
 
     def objenc_internal(x, depth=0)
@@ -255,7 +298,7 @@ module Lines
       if depth > max_depth
         '...'
       else
-        x.map{|k,v| "#{keyenc(k)}=#{valenc(v, depth)}" }.join(' ')
+        x.map{|k,v| "#{keyenc(k)}=#{valenc(v, depth)}" }.join(SPACE)
       end
     end
 
@@ -273,39 +316,40 @@ module Lines
       when Array          then arrenc(x, depth)
       when String, Symbol then strenc(x)
       when Numeric        then numenc(x)
-      when Time, Date     then timeenc(x)
-      when true           then '#t'
-      when false          then '#f'
-      when nil            then 'nil'
+      when Time           then timeenc(x)
+      when Date           then dateenc(x)
+      when true           then LIT_TRUE
+      when false          then LIT_FALSE
+      when nil            then LIT_NIL
       else
         litenc(x)
       end
     end
 
     def objenc(x, depth)
-      '{' + objenc_internal(x, depth) + '}'
+      OPEN_BRACE + objenc_internal(x, depth) + SHUT_BRACE
     end
 
     def arrenc(a, depth)
       depth += 1
       # num + unit. Eg: 3ms
       if a.size == 2 && a.first.kind_of?(Numeric) && is_literal?(a.last.to_s)
-        numenc(a.first) + strenc(a.last)
+        "#{numenc(a.first)}:#{strenc(a.last)}"
       elsif depth > max_depth
         '[...]'
       else
-        '[' + a.map{|x| valenc(x, depth)}.join(' ') + ']'
+        OPEN_BRACKET + a.map{|x| valenc(x, depth)}.join(' ') + SHUT_BRACKET
       end
     end
 
-    # TODO: Single-quote espace if possible
     def strenc(s)
       s = s.to_s
       unless is_literal?(s)
         s = s.inspect
-        unless s[1..-2].include?("'")
-          s[0] = s[-1] = "'"
-          s.gsub!('\"', '"')
+        unless s[1..-2].include?(SINGLE_QUOTE)
+          s.gsub!(SINGLE_QUOTE, "\\'")
+          s.gsub!('\"', DOUBLE_QUOTE)
+          s[0] = s[-1] = SINGLE_QUOTE
         end
       end
       s
@@ -336,8 +380,12 @@ module Lines
       t.utc.iso8601
     end
 
+    def dateenc(d)
+      d.iso8601
+    end
+
     def is_literal?(s)
-      !s.index(/[\s'"]/)
+      !s.index(/[\s'"=:{}\[\]]/)
     end
 
   end
@@ -360,3 +408,6 @@ module Lines
   end
   extend UniqueIDs
 end
+
+# default config
+Lines.use($stderr)

data/lib/lines/active_record.rb

@@ -1,4 +1,5 @@
 require 'active_record'
+require 'active_record/log_subscriber'
 require 'lines'
 
 module Lines

data/lib/lines/loader.rb

@@ -1,166 +1,229 @@
-begin
-  require 'parslet'
-rescue LoadError
-  warn "lines/loader depends on parslet"
-  raise
-end
-
-# http://zerowidth.com/2013/02/24/parsing-toml-in-ruby-with-parslet.html
 module Lines
   module Error; end
-  module ParseError; include Error; end
-  module Loader; extend self
-    def load(s)
-      parser = Parser.new
-      transformer = Transformer.new
-
-      tree = parser.parse(s)
-      #puts; p tree; puts
-      transformer.apply(tree)
-    rescue Parslet::ParseFailed => ex
-      # Mark as being part of the Lines library
-      ex.extend ParseError
-      raise
+
+  class Loader
+    class ParseError < StandardError; include Error; end
+
+    DOT                   = '.'
+    EQUAL                 = '='
+    SPACE                 = ' '
+    OPEN_BRACKET          = '['
+    SHUT_BRACKET          = ']'
+    OPEN_BRACE            = '{'
+    SHUT_BRACE            = '}'
+    SINGLE_QUOTE          = "'"
+    DOUBLE_QUOTE          = '"'
+    BACKSLASH             = '\\'
+
+    ESCAPED_SINGLE_QUOTE  = "\\'"
+    ESCAPED_DOUBLE_QUOTE  = '\"'
+
+    LITERAL_MATCH         = /[^=\s}\]]+/
+    SINGLE_QUOTE_MATCH    = /(?:\\.|[^'])*/
+    DOUBLE_QUOTE_MATCH    = /(?:\\.|[^"])*/
+
+    NUM_MATCH             = /-?(?:0|[1-9])\d*(?:\.\d+)?(?:[eE][+-]\d+)?/
+    ISO8601_ZULU_CAPTURE  = /^(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)Z$/
+    NUM_CAPTURE           = /^(#{NUM_MATCH})$/
+    UNIT_CAPTURE          = /^(#{NUM_MATCH}):(.+)/
+
+    # Speeds parsing up a bit
+    constants.each(&:freeze)
+
+    EOF                   = nil
+
+
+    def self.load(string)
+      new.parse(string)
     end
-  end
 
-  # Mostly copied over from the JSON example:
-  #   https://github.com/kschiess/parslet/blob/master/example/json.rb
-  #
-  # TODO:
-  #   ISO8601 dates
-  class Parser < Parslet::Parser
-
-    rule(:spaces) { match(' ').repeat(1) }
-    rule(:spaces?) { spaces.maybe }
-
-    rule(:digit) { match['0-9'] }
-
-    rule(:number) {
-      (
-        str('-').maybe >> (
-          str('0') | (match['1-9'] >> digit.repeat)
-        ) >> (
-          str('.') >> digit.repeat(1)
-        ).maybe >> (
-          match('[eE]') >> (str('+') | str('-')).maybe >> digit.repeat(1)
-        ).maybe
-      ).as(:number)
-    }
-
-    rule(:time) {
-      digit.repeat(4) >> str('-') >>
-      digit.repeat(2) >> str('-') >>
-      digit.repeat(2) >> str('T') >>
-      digit.repeat(2) >> str(':') >>
-      digit.repeat(2) >> str(':') >>
-      digit.repeat(2) >> str('Z')
-    }
-
-    rule(:singlequoted_string) {
-      str("'") >> (
-        str('\\') >> any | str("'").absent? >> any
-      ).repeat.as(:string) >> str("'")
-    }
-
-    rule(:doublequoted_string) {
-      str('"') >> (
-        str('\\') >> any | str('"').absent? >> any
-      ).repeat.as(:string) >> str('"')
-    }
-
-    rule(:simple_string) {
-      match['a-zA-Z_\-:'].repeat.as(:string)
-    }
-
-    rule(:string) {
-      singlequoted_string | doublequoted_string | simple_string
-    }
-
-    rule(:array) {
-      str('[') >> spaces? >>
-      (value >> (spaces >> value).repeat).maybe.as(:array) >>
-      spaces? >> str(']')
-    }
-
-    rule(:object) {
-      str('{') >> spaces? >>
-      (entry >> (spaces >> entry).repeat).maybe.as(:object) >>
-      spaces? >> str('}')
-    }
-
-    rule(:key) {
-      match['a-zA-Z0-9_'].repeat
-    }
-
-    rule(:value) {
-      str('#t').as(:true) | str('#f').as(:false) |
-      str('nil').as(:nil) |
-      object | array |
-      number | time |
-      string
-    }
-
-    rule(:entry) {
-      (
-         key.as(:key) >>
-         str('=') >>
-         value.as(:val)
-      ).as(:entry)
-    }
-
-    rule(:top) { spaces? >> (entry >> (spaces >> entry).repeat).maybe.as(:object) >> spaces? }
-    #rule(:top) { (digit >> digit).as(:digit) }
-    #rule(:top) { time }
-
-    root(:top)
-  end
+    def parse(string)
+      init(string.rstrip)
+      inner_obj
+    end
+
+    protected
+
+    def init(string)
+      @string = string
+      @pos = 0
+      @c = @string[0]
+    end
+
+    def getc
+      @pos += 1
+      @c = @string[@pos]
+    end
+
+    def accept(char)
+      if @c == char
+        getc
+        return true
+      end
+      false
+    end
+
+    def peek(num)
+      @string[@pos+num]
+    end
+
+    def skip(num)
+      @pos += num
+      @c = @string[@pos]
+    end
+
+    def match(reg)
+      @string.match(reg, @pos)
+    end
+
+    def expect(char)
+      if !accept(char)
+        fail "Expected '#{char}' but got '#{@c}'"
+      end
+    end
+
+    def fail(msg)
+      raise ParseError, "At #{@pos}, #{msg}"
+    end
+
+    def dbg(*x)
+      #p [@pos, @c, @string[0..@pos]] + x
+    end
+
+    # Structures
+
+
+    def inner_obj
+      dbg :inner_obj
+      # Shortcut for the '...' max_depth notation
+      if @c == DOT && peek(1) == DOT && peek(2) == DOT
+        expect DOT
+        expect DOT
+        expect DOT
+        return {'...' => ''}
+      end
+
+      return {} if @c == EOF || @c == SHUT_BRACE
 
-  class Transformer < Parslet::Transform
+      # First pair
+      k = key()
+      expect EQUAL
+      obj = {
+        k => value()
+      }
+
+      while accept(SPACE)
+        k = key()
+        expect EQUAL
+        obj[k] = value()
+      end
+
+      obj
+    end
 
-    class Entry < Struct.new(:key, :val); end
+    def key
+      dbg :key
 
-    rule(array: subtree(:ar)) {
-      case ar
-      when nil
-        []
-      when Array
-        ar
+      if @c == SINGLE_QUOTE
+        single_quoted_string
+      elsif @c == DOUBLE_QUOTE
+        double_quoted_string
       else
-        [ar]
+        literal(false)
       end
-    }
-    rule(object: subtree(:ob)) {
-      case ob
-      when nil
-        []
-      when Array
-        ob
+    end
+
+    def single_quoted_string
+      dbg :single_quoted_string
+
+      expect SINGLE_QUOTE
+      md = match SINGLE_QUOTE_MATCH
+      str = md[0].gsub ESCAPED_SINGLE_QUOTE, SINGLE_QUOTE
+      skip md[0].size
+
+      expect SINGLE_QUOTE
+      str
+    end
+
+    def double_quoted_string
+      dbg :double_quoted_string
+
+      expect DOUBLE_QUOTE
+      md = match DOUBLE_QUOTE_MATCH
+      str = md[0].gsub ESCAPED_DOUBLE_QUOTE, DOUBLE_QUOTE
+      skip md[0].size
+
+      expect DOUBLE_QUOTE
+      str
+    end
+
+    def literal(sub_parse)
+      dbg :literal, sub_parse
+
+      return "" unless ((md = match LITERAL_MATCH))
+
+      literal = md[0]
+      skip literal.size
+      
+      return literal unless sub_parse
+
+      case literal
+      when 'nil'
+        nil
+      when '#t'
+        true
+      when '#f'
+        false
+      when ISO8601_ZULU_CAPTURE
+        Time.new($1.to_i, $2.to_i, $3.to_i, $4.to_i, $5.to_i, $6.to_i, '+00:00').utc
+      when NUM_CAPTURE
+        literal.index('.') ? Float(literal) : Integer(literal)
+      when UNIT_CAPTURE
+        num = $1.index('.') ? Float($1) : Integer($1)
+        unit = $2
+        [num, unit]
       else
-        [ob]
-      end.inject({}) { |h, e|
-        h[e[:entry][:key].to_s] = e[:entry][:val]; h
-      }
-    }
+        literal
+      end
+    end
 
-    # rule(entry: { key: simple(:ke), val: simple(:va) }) {
-    #   Entry.new(ke.to_s, va)
-    # }
+    def value
+      dbg :value
+
+      case @c
+      when OPEN_BRACKET
+        list
+      when OPEN_BRACE
+        object
+      when DOUBLE_QUOTE
+        double_quoted_string
+      when SINGLE_QUOTE
+        single_quoted_string
+      else
+        literal(:sub_parse)
+      end
+    end
 
-    rule(time: { year: simple(:ye), month: simple(:mo), day: simple(:da), hour: simple(:ho), minute: simple(:min), second: simple(:sec)}) {
-      Time.new(ye.to_i, mo.to_i, da.to_i, ho.to_i, min.to_i, sec.to_i, "+00:00")
-    }
+    def list
+      dbg :list
 
-    rule(string: simple(:st)) {
-      st.to_s
-    }
+      list = []
+      expect(OPEN_BRACKET)
+      list.push value
+      while accept(SPACE)
+        list.push value
+      end
+      expect(SHUT_BRACKET)
+      list
+    end
 
-    rule(number: simple(:nb)) {
-      nb.match(/[eE\.]/) ? Float(nb) : Integer(nb)
-    }
+    def object
+      dbg :object
 
-    rule(nil: simple(:ni)) { nil }
-    rule(true: simple(:tr)) { true }
-    rule(false: simple(:fa)) { false }
+      expect(OPEN_BRACE)
+      obj = inner_obj
+      expect(SHUT_BRACE)
+      obj
+    end
   end
 end

data/lib/lines/version.rb

@@ -1,3 +1,3 @@
 module Lines
-  VERSION = "0.1.27"
+  VERSION = "0.2.0"
 end

data/spec/bench.rb

@@ -1,56 +1,46 @@
+require 'benchmark/ips'
+
 $:.unshift File.expand_path('../../lib', __FILE__)
 require 'lines'
 
-module Kernel
-  def bm(name = nil, &what)
-    start = Time.now.to_f
-    count = 0
-    max = 0.5
-    name ||= what.source_location.join(':')
-    $stdout.write "#{name} : "
-    while Time.now.to_f - start < max
-      yield
-      count += 1
-    end
-    $stdout.puts "%0.3f fps" % (count / max)
-  end
-end
-
 class FakeIO
   def write(*a)
   end
   alias syswrite write
 end
 
-Lines.global[:app] = 'benchmark'
-Lines.global[:at] = proc{ Time.now }
-Lines.global[:pid] = Process.pid
+globals = {
+  app: 'benchmark',
+  at: proc{ Time.now },
+  pid: Process.pid,
+}
 
 EX = (raise "FOO" rescue $!)
 
-Lines.use FakeIO.new
-bm "FakeIO write" do
-  Lines.log EX
-end
+Benchmark.ips do |x|
+  x.report "FakeIO write" do |n|
+    Lines.use(FakeIO.new, globals)
+    n.times{ Lines.log EX }
+  end
 
-dev_null = File.open('/dev/null', 'w')
-Lines.use dev_null
-bm "/dev/null write" do
-  Lines.log EX
-end
+  x.report "/dev/null write" do |n|
+    dev_null = File.open('/dev/null', 'w')
+    Lines.use(dev_null, globals)
+    n.times{ Lines.log EX }
+  end
 
-Lines.use Syslog
-bm "syslog write" do
-  Lines.log EX
-end
+  x.report "syslog write" do |n|
+    Lines.use(Syslog, globals)
+    n.times{ Lines.log EX }
+  end
 
-real_file = File.open('real_file.log', 'w')
-Lines.use real_file
-bm "real file" do
-  Lines.log EX
-end
+  x.report "real file" do |n|
+    real_file = File.open('real_file.log', 'w')
+    Lines.use(real_file, globals)
+    n.times{ Lines.log EX }
+  end
 
-bm "real file logger" do
-  Lines.logger.info "Ahoi this is a really cool option"
+  x.report "real file logger" do |n|
+    n.times{ Lines.logger.info "Ahoi this is a really cool option" }
+  end
 end
-

data/spec/lines_loader_spec.rb

@@ -1,61 +1,55 @@
 require 'spec_helper'
 require 'lines/loader'
 
-module Lines
-  describe Loader do
-    subject { Loader }
-
-    it "can load stuff" do
-      expect(Loader.load 'foo=bar').to eq("foo" => "bar")
-    end
-  end
-
-  describe Parser do
-    let(:parser) { Lines::Parser.new }
-
-    context "value parsing" do
-      let(:value_parser) { parser.value }
-
-      it "parses integers" do
-        pending
-        expect(value_parser).to     parse("1")
-        expect(value_parser).to     parse("-123")
-        expect(value_parser).to     parse("120381")
-        expect(value_parser).to     parse("181")
-      end
-
-      it "parses floats" do
-        pending
-        expect(value_parser).to     parse("0.1")
-        expect(value_parser).to     parse("3.14159")
-        expect(value_parser).to     parse("-0.00001")
-      end
-
-      it "parses booleans" do
-        pending
-        expect(value_parser).to     parse("#t")
-        expect(value_parser).to     parse("#f")
-      end
-
-      it "parses datetimes" do
-        pending
-        expect(value_parser).to     parse("1979-05-27T07:32:00Z")
-        expect(value_parser).to     parse("2013-02-24T17:26:21Z")
-        expect(value_parser).to_not parse("1979l05-27 07:32:00")
-      end
-
-      it "parses strings" do
-        pending
-        expect(value_parser).to     parse('""')
-        expect(value_parser).to     parse('"hello world"')
-        expect(value_parser).to     parse('"hello\\nworld"')
-        expect(value_parser).to     parse('"hello\\t\\n\\\\\\0world\\n"')
-        expect(value_parser).to_not parse("\"hello\nworld\"")
-      end
-    end
-  end
-
-  describe Transformer do
+describe Lines::Loader do
+  subject { Lines::Loader.new }
 
+  def expect_load(str)
+    expect(subject.parse str)
+  end
+
+  it "can load stuff" do
+    expect_load('foo=bar bar=33').to eq("foo" => "bar", "bar" => 33)
+  end
+
+  it "handles max_depth items" do
+    expect_load('x=[...]').to eq("x" => ["..."])
+    expect_load('x={...}').to eq("x" => {"..." => ""})
+  end
+
+  it "treats missing value in a pair as an empty string" do
+    expect_load('x=').to eq("x" => "")
+  end
+
+  it "has non-greedy string parsing" do
+    expect_load('x="foo" bar="baz"').to eq("x" => "foo", "bar" => "baz")
+  end
+
+  it "unscapes quotes in quoted strings" do
+    expect_load("x='foo\\'bar'").to eq("x" => "foo'bar")
+    expect_load('x="foo\"bar"').to eq("x" => 'foo"bar')
+  end
+
+  it "doesn't parse literals when they are keys" do
+    expect_load("3=4").to eq("3" => 4)
+  end
+
+  it "handles some random stuff" do
+    expect_load("=").to eq("" => "")
+    expect_load('"\""=zzz').to eq('"' => "zzz")
+  end
+
+  it "parses sample log lines" do
+    expect_load("commit=716f337").to eq("commit" => "716f337")
+
+    line = <<LINE
+at=2013-07-12T21:33:47Z commit=716f337 sql="SELECT FROM_UNIXTIME(UNIX_TIMESTAMP(created_at) - UNIX_TIMESTAMP(created_at)%(300)) as timestamp FROM `job_queue_logs` WHERE `job_queue_logs`.`account_id` = 'effe376baf553c590c02090abe512278' AND (created_at >= '2013-06-28 16:56:12') GROUP BY timestamp" elapsed=31.9:ms
+LINE
+    expect_load(line).to eq(
+      "at" => Time.at(1373664827).utc,
+      "commit" => "716f337",
+      "sql" => "SELECT FROM_UNIXTIME(UNIX_TIMESTAMP(created_at) - UNIX_TIMESTAMP(created_at)%(300)) as timestamp FROM `job_queue_logs` WHERE `job_queue_logs`.`account_id` = 'effe376baf553c590c02090abe512278' AND (created_at >= '2013-06-28 16:56:12') GROUP BY timestamp",
+      "elapsed" => [31.9, "ms"],
+    )
   end
 end

data/spec/lines_spec.rb

@@ -2,28 +2,29 @@ require 'spec_helper'
 require 'lines'
 require 'stringio'
 
+NL = "\n"
+
 describe Lines do
   let(:outputter) { StringIO.new }
   let(:output) { outputter.string }
   before do
-    Lines.global.replace({})
     Lines.use(outputter)
   end
 
   context ".log" do
     it "logs stuff" do
       Lines.log(foo: 'bar')
-      expect(output).to eq('foo=bar' + Lines::NL)
+      expect(output).to eq('foo=bar' + NL)
     end
 
     it "supports a first msg argument" do
       Lines.log("this user is annoying", user: 'bob')
-      expect(output).to eq("msg='this user is annoying' user=bob" + Lines::NL)
+      expect(output).to eq("msg='this user is annoying' user=bob" + NL)
     end
 
     it "logs exceptions" do
       Lines.log(StandardError.new("error time!"), user: 'bob')
-      expect(output).to eq("ex=StandardError msg='error time!' user=bob" + Lines::NL)
+      expect(output).to eq("ex=StandardError msg='error time!' user=bob" + NL)
     end
 
     it "logs exception backtraces when available" do
@@ -35,33 +36,33 @@ describe Lines do
 
     it "works with anything" do
       Lines.log("anything1", "anything2")
-      expect(output).to eq('msg=anything2' + Lines::NL)
+      expect(output).to eq('msg=anything2' + NL)
     end
 
     it "doesn't convert nil args to msg" do
       Lines.log("anything", nil)
-      expect(output).to eq('msg=anything' + Lines::NL)
+      expect(output).to eq('msg=anything' + NL)
     end
   end
 
   context ".context" do
     it "has contextes" do
       Lines.context(foo: "bar").log(a: 'b')
-      expect(output).to eq('a=b foo=bar' + Lines::NL)
+      expect(output).to eq('a=b foo=bar' + NL)
     end
 
     it "has contextes with blocks" do
       Lines.context(foo: "bar") do |ctx|
         ctx.log(a: 'b')
       end
-      expect(output).to eq('a=b foo=bar' + Lines::NL)
+      expect(output).to eq('a=b foo=bar' + NL)
     end
 
     it "mixes everything" do
       Lines.global[:app] = :self
       ctx = Lines.context(foo: "bar")
       ctx.log('msg', ahoi: true)
-      expect(output).to eq('app=self msg=msg ahoi=#t foo=bar' + Lines::NL)
+      expect(output).to eq('app=self msg=msg ahoi=#t foo=bar' + NL)
     end
   end
 
@@ -69,7 +70,7 @@ describe Lines do
     it "is provided for backward-compatibility" do
       l = Lines.logger
       l.info("hi")
-      expect(output).to eq('pri=info msg=hi' + Lines::NL)
+      expect(output).to eq('pri=info msg=hi' + NL)
     end
   end
 
@@ -77,7 +78,7 @@ describe Lines do
     it "prepends data to the line" do
       Lines.global["app"] = :self
       Lines.log 'hey'
-      expect(output).to eq('app=self msg=hey' + Lines::NL)
+      expect(output).to eq('app=self msg=hey' + NL)
     end
 
     it "resolves procs dynamically" do
@@ -86,15 +87,15 @@ describe Lines do
       Lines.log 'test1'
       Lines.log 'test2'
       expect(output).to eq(
-        'count=1 msg=test1' + Lines::NL +
-        'count=2 msg=test2' + Lines::NL
+        'count=1 msg=test1' + NL +
+        'count=2 msg=test2' + NL
       )
     end
 
     it "doesn't fail if a proc has an exception" do
       Lines.global[:X] = proc{ fail "error" }
       Lines.log 'test'
-      expect(output).to eq("X='#<RuntimeError: error>' msg=test" + Lines::NL)
+      expect(output).to eq("X='#<RuntimeError: error>' msg=test" + NL)
     end
   end
 end
@@ -128,18 +129,22 @@ describe Lines::Dumper do
   end
 
   it "can dump a basicobject" do
-    expect_dump(foo: BasicObject.new).to match(/foo=#<BasicObject:0x[0-9a-f]+>/)
+    expect_dump(foo: BasicObject.new).to match(/foo='#<BasicObject:0x[0-9a-f]+>'/)
   end
 
   it "can dump IO objects" do
     expect_dump(foo: File.open(__FILE__)).to match(/foo='?#<File:[^>]+>'?/)
-    expect_dump(foo: STDOUT).to match(/^foo=(?:#<IO:<STDOUT>>|'#<IO:fd 1>')$/)
+    expect_dump(foo: STDOUT).to match(/^foo='(?:#<IO:<STDOUT>>|#<IO:fd 1>)'$/)
   end
 
   it "dumps time as ISO zulu format" do
     expect_dump(foo: Time.at(1337)).to eq('foo=1970-01-01T00:22:17Z')
   end
 
+  it "dumps date as ISO date" do
+    expect_dump(foo: Date.new(1968, 3, 7)).to eq('foo=1968-03-07')
+  end
+
   it "dumps symbols as strings" do
     expect_dump(foo: :some_symbol).to eq('foo=some_symbol')
     expect_dump(foo: :"some symbol").to eq("foo='some symbol'")
@@ -158,14 +163,14 @@ describe Lines::Dumper do
   end
 
   it "dumps [number, literal] tuples as numberliteral" do
-    expect_dump(foo: [3, :ms]).to eq('foo=3ms')
-    expect_dump(foo: [54.2, 's']).to eq('foo=54.2s')
+    expect_dump(foo: [3, :ms]).to eq('foo=3:ms')
+    expect_dump(foo: [54.2, 's']).to eq('foo=54.2:s')
   end
 
   it "knows how to handle circular dependencies" do
     x = {}
     x[:x] = x
-    expect_dump(x).to eq('x={x={x={...}}}')
+    expect_dump(x).to eq('x={x={x={x={...}}}}')
   end
 end
 

data/spec/parse-bench

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+
+libdir = File.expand_path('../../lib', __FILE__)
+$:.unshift(libdir) unless $:.include? libdir
+
+require 'lines'
+
+loader = Lines.loader
+
+start = Time.now
+line_count = 0
+
+$stdin.lines.each do |line|
+  begin
+    line_count += 1
+    loader.load(line)
+  rescue Lines::Error => ex
+    # Lines seem to get truncated by syslog when too long
+    if line.size < 2000
+      p line
+      p ex
+    end
+  end
+end
+
+puts "Parsed #{line_count / (Time.now - start)} lines per second"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lines
 version: !ruby/object:Gem::Version
-  version: 0.1.27
+  version: 0.2.0
 platform: ruby
 authors:
 - Jonas Pfenniger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-10 00:00:00.000000000 Z
+date: 2013-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -63,6 +63,7 @@ files:
 - spec/bench.rb
 - spec/lines_loader_spec.rb
 - spec/lines_spec.rb
+- spec/parse-bench
 - spec/spec_helper.rb
 homepage: https://github.com/zimbatm/lines-ruby
 licenses:
@@ -92,4 +93,5 @@ test_files:
 - spec/bench.rb
 - spec/lines_loader_spec.rb
 - spec/lines_spec.rb
+- spec/parse-bench
 - spec/spec_helper.rb