lines 0.2.0 → 0.9.1

data/spec/lines_generator_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+require 'lines/generator'
+
+describe Lines::Generator do
+  def expect_dump(obj, opts={})
+    expect(subject.generate obj, opts)
+  end
+
+  it "can dump stuff" do
+    expect_dump(foo: "bar", bar: 33).to eq('foo=bar bar=33')
+  end
+
+  it "handles max_depth items" do
+    expect_dump({x: [444]}, max_nesting: 1).to eq('x=[...]')
+    expect_dump({x: {y: 444}}, max_nesting: 1).to eq('x={...}')
+  end
+
+  it "handles max_size items" do
+    expect_dump({a: "aaa", b: "bbb", c: "cccc"}, max_bytesize: 18).to eq('a=aaa b=bbb c=cccc')
+    expect_dump({a: "aaa", b: "bbb", c: "cccc"}, max_bytesize: 17).to eq('a=aaa b=bbb ...')
+    expect_dump({a: "aaa", b: "bbb", c: "cccc"}, max_bytesize: 15).to eq('a=aaa b=bbb ...')
+    expect_dump({a: "aaa", b: "bbb", c: "cccc"}, max_bytesize: 14).to eq('a=aaa ...')
+  end
+
+  it "treats missing value in a pair as an empty string" do
+    expect_dump(x: '').to eq("x=")
+  end
+
+  it "escapes quotes in quoted strings" do
+    expect_dump(x: "foo'bar").to eq("x=\"foo'bar\"")
+    expect_dump(x: 'foo"bar').to eq("x='foo\"bar'")
+  end
+
+  it "doesn't parse literals when they are keys" do
+    expect_dump(3 => 4).to eq("3=4")
+  end
+
+  it "escapes ambiguous strings" do
+    expect_dump(x: '4', y: '-3.3').to eq("x='4' y='-3.3'")
+    expect_dump(a: '#t', b: '#f', c: 'nil').to eq("a='#t' b='#f' c='nil'")
+  end
+
+  it "handles some random stuff" do
+    expect_dump("" => "").to eq("=")
+    expect_dump('"' => 'zzz').to eq('\'"\'=zzz')
+  end
+
+  it "escapes various whitespace characters" do
+    expect_dump("\r\n\t" => "\r\n\t").to eq('\'\r\n\t\'=\'\r\n\t\'')
+  end
+
+  it "generates sample log lines" do
+    expect_dump("commit" => "716f337").to eq("commit=716f337")
+
+    line = <<LINE.rstrip
+at=2013-07-12T21:33:47Z commit=716f337 sql="SELECT FROM_UNIXTIME(UNIX_TIMESTAMP(created_at) - UNIX_TIMESTAMP(created_at)%(300)) as timestamp FROM `job_queue_logs` WHERE `job_queue_logs`.`account_id` = 'effe376baf553c590c02090abe512278' AND (created_at >= '2013-06-28 16:56:12') GROUP BY timestamp" elapsed=[31.9 ms]
+LINE
+    expect_dump(
+      "at" => Time.at(1373664827).utc,
+      "commit" => "716f337",
+      "sql" => "SELECT FROM_UNIXTIME(UNIX_TIMESTAMP(created_at) - UNIX_TIMESTAMP(created_at)%(300)) as timestamp FROM `job_queue_logs` WHERE `job_queue_logs`.`account_id` = 'effe376baf553c590c02090abe512278' AND (created_at >= '2013-06-28 16:56:12') GROUP BY timestamp",
+      "elapsed" => [31.9, "ms"],
+    ).to eq(line)
+  end
+end

data/spec/lines_parser_bench.rb

@@ -0,0 +1,50 @@
+$:.unshift File.expand_path('../../lib', __FILE__)
+
+require 'benchmark/ips'
+require 'time'
+
+$message = {
+  "at" => Time.now.utc.iso8601,
+  "pid" => Process.pid,
+  "app" => File.basename($0),
+  "pri" => "info",
+  "msg" => "This is my message",
+  "user" => {"t" => true, "f" => false, "n" => nil},
+  "elapsed" => [55.67, 'ms'],
+}
+
+$data = {}
+
+formatters = [
+  ['lines', "Lines.dump($message)", "Lines.load($data[:lines])"],
+
+  ['json/pure', "JSON.dump($message)", "JSON.load($data[:'json/pure'])"],
+  ['oj', "Oj.dump($message)", "Oj.load($data[:oj])"],
+  ['yajl', "Yajl.dump($message)", "Yajl.load($data[:yajl])"],
+  
+  ['msgpack', "MessagePack.dump($message)", "MessagePack.load($data[:msgpack])"],
+  ['bson', "$message.to_bson", "???"],
+  ['tnetstring', "TNetstring.dump($message)", "TNetstring.load($data[:tnetstring])"],
+]
+
+Benchmark.ips do |x|
+  x.compare!
+  formatters.each do |(feature, dumper, loader)|
+    begin
+      require feature
+
+      $data[feature.to_sym] = eval dumper
+      #puts "%-12s %-5d %s" % [feature, data.size, data]
+
+      msg = eval loader
+
+      if $message != msg
+        p [:invalid, $message, msg]
+      end
+
+      x.report feature, loader
+    rescue LoadError
+      puts "%-12s could not be loaded" % [feature]
+    end
+  end
+end

data/spec/{lines_loader_spec.rb → lines_parser_spec.rb}

@@ -1,11 +1,11 @@
 require 'spec_helper'
-require 'lines/loader'
+require 'lines/parser'
 
-describe Lines::Loader do
-  subject { Lines::Loader.new }
+describe Lines::Parser do
+  subject{ Lines::Parser }
 
-  def expect_load(str)
-    expect(subject.parse str)
+  def expect_load(str, opts={})
+    expect(subject.parse str, opts)
   end
 
   it "can load stuff" do
@@ -17,12 +17,20 @@ describe Lines::Loader do
     expect_load('x={...}').to eq("x" => {"..." => ""})
   end
 
+  it "handles unfinished lines" do
+    expect_load('x=foo ').to eq("x" => "foo")
+  end
+
+  it "handles \\n at the end of line" do
+    expect_load("x=foo\n").to eq("x" => "foo")
+  end
+
   it "treats missing value in a pair as an empty string" do
     expect_load('x=').to eq("x" => "")
   end
 
   it "has non-greedy string parsing" do
-    expect_load('x="foo" bar="baz"').to eq("x" => "foo", "bar" => "baz")
+    expect_load('x="#t" bar="baz"').to eq("x" => "#t", "bar" => "baz")
   end
 
   it "unscapes quotes in quoted strings" do
@@ -39,11 +47,24 @@ describe Lines::Loader do
     expect_load('"\""=zzz').to eq('"' => "zzz")
   end
 
+  it "knows how to restore iso time" do
+    expect_load("at=2013-07-12T21:33:47Z").to eq("at" => Time.at(1373664827).utc)
+  end
+
+  it "can symbolize the names" do
+    expect_load("foo=bar", symbolize_names: true).to eq(foo: "bar")
+  end
+
+  it "restores escaped characters from a string" do
+    pending
+    expect_load('\'\r\n\t\'=\'\r\n\t\'').to eq("\r\n\t" => "\r\n\t")
+  end
+
   it "parses sample log lines" do
     expect_load("commit=716f337").to eq("commit" => "716f337")
 
     line = <<LINE
-at=2013-07-12T21:33:47Z commit=716f337 sql="SELECT FROM_UNIXTIME(UNIX_TIMESTAMP(created_at) - UNIX_TIMESTAMP(created_at)%(300)) as timestamp FROM `job_queue_logs` WHERE `job_queue_logs`.`account_id` = 'effe376baf553c590c02090abe512278' AND (created_at >= '2013-06-28 16:56:12') GROUP BY timestamp" elapsed=31.9:ms
+at=2013-07-12T21:33:47Z commit=716f337 sql="SELECT FROM_UNIXTIME(UNIX_TIMESTAMP(created_at) - UNIX_TIMESTAMP(created_at)%(300)) as timestamp FROM `job_queue_logs` WHERE `job_queue_logs`.`account_id` = 'effe376baf553c590c02090abe512278' AND (created_at >= '2013-06-28 16:56:12') GROUP BY timestamp" elapsed=[31.9 ms]
 LINE
     expect_load(line).to eq(
       "at" => Time.at(1373664827).utc,

data/spec/spec_helper.rb

@@ -1,17 +1,13 @@
 # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
 RSpec.configure do |config|
-  config.treat_symbols_as_metadata_keys_with_true_values = true
-  config.run_all_when_everything_filtered = true
-  config.filter_run :focus
-
+  # Only use the expect syntax
   config.expect_with :rspec do |c|
     c.syntax = :expect
   end
 
-
   # Run specs in random order to surface order dependencies. If you find an
   # order dependency and want to debug it, you can fix the order by providing
   # the seed, which is printed after each run.
   #     --seed 1234
   config.order = 'random'
-end
+end

metadata

@@ -1,69 +1,98 @@
 --- !ruby/object:Gem::Specification
 name: lines
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.9.1
 platform: ruby
 authors:
-- Jonas Pfenniger
+- zimbatm
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-15 00:00:00.000000000 Z
+date: 2014-11-12 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: structured logs for humans
+description: |
+  A log format that's readable by humans and easily parseable by computers.
 email:
-- jonas@pfenniger.name
+- zimbatm@zimbatm.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
-- .travis.yml
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/cli.rb
 - lib/lines.rb
-- lib/lines/active_record.rb
-- lib/lines/loader.rb
-- lib/lines/logger.rb
-- lib/lines/rack_logger.rb
+- lib/lines/common.rb
+- lib/lines/generator.rb
+- lib/lines/parser.rb
 - lib/lines/version.rb
 - lines.gemspec
-- spec/bench.rb
-- spec/lines_loader_spec.rb
-- spec/lines_spec.rb
-- spec/parse-bench
+- spec/lines_generator_bench.rb
+- spec/lines_generator_spec.rb
+- spec/lines_parser_bench.rb
+- spec/lines_parser_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/zimbatm/lines-ruby
 licenses:
@@ -75,23 +104,23 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.4.2
 signing_key: 
 specification_version: 4
-summary: Lines is an opinionated structured logging library
+summary: Lines is an opinionated structured log format
 test_files:
-- spec/bench.rb
-- spec/lines_loader_spec.rb
-- spec/lines_spec.rb
-- spec/parse-bench
+- spec/lines_generator_bench.rb
+- spec/lines_generator_spec.rb
+- spec/lines_parser_bench.rb
+- spec/lines_parser_spec.rb
 - spec/spec_helper.rb

data/lib/lines/active_record.rb

@@ -1,70 +0,0 @@
-require 'active_record'
-require 'active_record/log_subscriber'
-require 'lines'
-
-module Lines
-  class ActiveRecordSubscriber < ActiveSupport::LogSubscriber
-    def render_bind(column, value)
-      if column
-        if column.binary?
-          value = "<#{value.bytesize} bytes of binary data>"
-        end
-
-        [column.name, value]
-      else
-        [nil, value]
-      end
-    end
-
-    def sql(event)
-      payload = event.payload
-
-      return if payload[:name] == 'SCHEMA' || payload[:name] == 'EXPLAIN'
-
-      args = {}
-
-      args[:name] = payload[:name] if payload[:name]
-      args[:sql] = payload[:sql].squeeze(' ')
-
-      if payload[:binds] && payload[:binds].any?
-        args[:binds] = payload[:binds].inject({}) do |hash,(col, v)|
-          k, v = render_bind(col, v)
-          hash[k] = v
-          hash
-        end
-      end
-
-      args[:elapsed] = [event.duration.round(1), 'ms']
-
-      Lines.log(args)
-    end
-
-    def identity(event)
-      Lines.log(name: event.payload[:name], line: event.payload[:line])
-    end
-
-    def logger; Lines.logger; end
-  end
-end
-
-# Subscribe lines to the AR events
-Lines::ActiveRecordSubscriber.attach_to :active_record
-
-# Remove the default ActiveRecord::LogSubscriber to avoid double outputs
-ActiveSupport::LogSubscriber.log_subscribers.each do |subscriber|
-  if subscriber.is_a?(ActiveRecord::LogSubscriber)
-    component = :active_record
-    events = subscriber.public_methods(false).reject{ |method| method.to_s == 'call' }
-    events.each do |event|
-      ActiveSupport::Notifications.notifier.listeners_for("#{event}.#{component}").each do |listener|
-        if listener.instance_variable_get('@delegate') == subscriber
-          ActiveSupport::Notifications.unsubscribe listener
-        end
-      end
-    end
-  end
-end
-
-# Make sure it has a logger just in case
-ActiveRecord::Base.logger = Lines.logger
-

data/lib/lines/loader.rb

@@ -1,229 +0,0 @@
-module Lines
-  module Error; end
-
-  class Loader
-    class ParseError < StandardError; include Error; end
-
-    DOT                   = '.'
-    EQUAL                 = '='
-    SPACE                 = ' '
-    OPEN_BRACKET          = '['
-    SHUT_BRACKET          = ']'
-    OPEN_BRACE            = '{'
-    SHUT_BRACE            = '}'
-    SINGLE_QUOTE          = "'"
-    DOUBLE_QUOTE          = '"'
-    BACKSLASH             = '\\'
-
-    ESCAPED_SINGLE_QUOTE  = "\\'"
-    ESCAPED_DOUBLE_QUOTE  = '\"'
-
-    LITERAL_MATCH         = /[^=\s}\]]+/
-    SINGLE_QUOTE_MATCH    = /(?:\\.|[^'])*/
-    DOUBLE_QUOTE_MATCH    = /(?:\\.|[^"])*/
-
-    NUM_MATCH             = /-?(?:0|[1-9])\d*(?:\.\d+)?(?:[eE][+-]\d+)?/
-    ISO8601_ZULU_CAPTURE  = /^(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)Z$/
-    NUM_CAPTURE           = /^(#{NUM_MATCH})$/
-    UNIT_CAPTURE          = /^(#{NUM_MATCH}):(.+)/
-
-    # Speeds parsing up a bit
-    constants.each(&:freeze)
-
-    EOF                   = nil
-
-
-    def self.load(string)
-      new.parse(string)
-    end
-
-    def parse(string)
-      init(string.rstrip)
-      inner_obj
-    end
-
-    protected
-
-    def init(string)
-      @string = string
-      @pos = 0
-      @c = @string[0]
-    end
-
-    def getc
-      @pos += 1
-      @c = @string[@pos]
-    end
-
-    def accept(char)
-      if @c == char
-        getc
-        return true
-      end
-      false
-    end
-
-    def peek(num)
-      @string[@pos+num]
-    end
-
-    def skip(num)
-      @pos += num
-      @c = @string[@pos]
-    end
-
-    def match(reg)
-      @string.match(reg, @pos)
-    end
-
-    def expect(char)
-      if !accept(char)
-        fail "Expected '#{char}' but got '#{@c}'"
-      end
-    end
-
-    def fail(msg)
-      raise ParseError, "At #{@pos}, #{msg}"
-    end
-
-    def dbg(*x)
-      #p [@pos, @c, @string[0..@pos]] + x
-    end
-
-    # Structures
-
-
-    def inner_obj
-      dbg :inner_obj
-      # Shortcut for the '...' max_depth notation
-      if @c == DOT && peek(1) == DOT && peek(2) == DOT
-        expect DOT
-        expect DOT
-        expect DOT
-        return {'...' => ''}
-      end
-
-      return {} if @c == EOF || @c == SHUT_BRACE
-
-      # First pair
-      k = key()
-      expect EQUAL
-      obj = {
-        k => value()
-      }
-
-      while accept(SPACE)
-        k = key()
-        expect EQUAL
-        obj[k] = value()
-      end
-
-      obj
-    end
-
-    def key
-      dbg :key
-
-      if @c == SINGLE_QUOTE
-        single_quoted_string
-      elsif @c == DOUBLE_QUOTE
-        double_quoted_string
-      else
-        literal(false)
-      end
-    end
-
-    def single_quoted_string
-      dbg :single_quoted_string
-
-      expect SINGLE_QUOTE
-      md = match SINGLE_QUOTE_MATCH
-      str = md[0].gsub ESCAPED_SINGLE_QUOTE, SINGLE_QUOTE
-      skip md[0].size
-
-      expect SINGLE_QUOTE
-      str
-    end
-
-    def double_quoted_string
-      dbg :double_quoted_string
-
-      expect DOUBLE_QUOTE
-      md = match DOUBLE_QUOTE_MATCH
-      str = md[0].gsub ESCAPED_DOUBLE_QUOTE, DOUBLE_QUOTE
-      skip md[0].size
-
-      expect DOUBLE_QUOTE
-      str
-    end
-
-    def literal(sub_parse)
-      dbg :literal, sub_parse
-
-      return "" unless ((md = match LITERAL_MATCH))
-
-      literal = md[0]
-      skip literal.size
-      
-      return literal unless sub_parse
-
-      case literal
-      when 'nil'
-        nil
-      when '#t'
-        true
-      when '#f'
-        false
-      when ISO8601_ZULU_CAPTURE
-        Time.new($1.to_i, $2.to_i, $3.to_i, $4.to_i, $5.to_i, $6.to_i, '+00:00').utc
-      when NUM_CAPTURE
-        literal.index('.') ? Float(literal) : Integer(literal)
-      when UNIT_CAPTURE
-        num = $1.index('.') ? Float($1) : Integer($1)
-        unit = $2
-        [num, unit]
-      else
-        literal
-      end
-    end
-
-    def value
-      dbg :value
-
-      case @c
-      when OPEN_BRACKET
-        list
-      when OPEN_BRACE
-        object
-      when DOUBLE_QUOTE
-        double_quoted_string
-      when SINGLE_QUOTE
-        single_quoted_string
-      else
-        literal(:sub_parse)
-      end
-    end
-
-    def list
-      dbg :list
-
-      list = []
-      expect(OPEN_BRACKET)
-      list.push value
-      while accept(SPACE)
-        list.push value
-      end
-      expect(SHUT_BRACKET)
-      list
-    end
-
-    def object
-      dbg :object
-
-      expect(OPEN_BRACE)
-      obj = inner_obj
-      expect(SHUT_BRACE)
-      obj
-    end
-  end
-end