lines 0.2.0 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,37 @@
1
+ # -*- encoding : utf-8 -*-
2
+
3
+ module Lines
4
+ module Error
5
+ # Used to mark non-lines errors as being part of the library. This lets
6
+ # a library user `rescue Lines::Error => ex` and catch all exceptions
7
+ # comming from the lines library.
8
+ def self.tag(obj)
9
+ obj.extend Error
10
+ end
11
+ end
12
+
13
+ class ParseError < StandardError; include Error; end
14
+ #class LogicError < RuntimeError; include Error; end
15
+
16
+ LIT_TRUE = '#t'
17
+ LIT_FALSE = '#f'
18
+ LIT_NIL = 'nil'
19
+
20
+ SPACE = ' '
21
+ EQUAL = '='
22
+ OPEN_BRACE = '{'
23
+ SHUT_BRACE = '}'
24
+ OPEN_BRACKET = '['
25
+ SHUT_BRACKET = ']'
26
+ SINGLE_QUOTE = "'"
27
+ DOUBLE_QUOTE = '"'
28
+ DOT_DOT_DOT = '...'
29
+
30
+ BACKSLASH = '\\'
31
+ ESCAPED_SINGLE_QUOTE = "\\'"
32
+ ESCAPED_DOUBLE_QUOTE = '\"'
33
+
34
+ NUM_MATCH = /-?(?:0|[1-9])\d*(?:\.\d+)?(?:[eE][+-]\d+)?/
35
+ ISO8601_ZULU_CAPTURE = /^(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)Z$/
36
+ NUM_CAPTURE = /^(#{NUM_MATCH})$/
37
+ end
@@ -0,0 +1,168 @@
1
+ # -*- encoding : utf-8 -*-
2
+
3
+ require 'time'
4
+
5
+ require 'lines/common'
6
+
7
+ module Lines
8
+ # Some opinions here as well on the format:
9
+ #
10
+ # We really want to never fail at dumping because you know, they're logs.
11
+ # It's better to get a slightly less readable log that no logs at all.
12
+ #
13
+ # We're trying to be helpful for humans. It means that if possible we want
14
+ # to make things shorter and more readable. It also means that ideally
15
+ # we would like the parsing to be isomorphic but approximations are alright.
16
+ # For example a symbol might become a string.
17
+ #
18
+ # Basically, values are either composite (dictionaries and arrays), quoted
19
+ # strings or litterals. Litterals are strings that can be parsed to
20
+ # something else depending if the language supports it or not.
21
+ # Litterals never contain white-spaces or other weird (very precise !) characters.
22
+ #
23
+ # the true litteral is written as "#t"
24
+ # the false litteral is written as "#f"
25
+ # the nil / null litteral is written as "nil"
26
+ #
27
+ # dictionary keys are always strings or litterals.
28
+ #
29
+ # Pleaaase, keep units with numbers. And we provide a way for this:
30
+ # a tuple of (number, litteral) can be concatenated. Eg: (3, 'ms') => 3:ms
31
+ # alternatively if your language supports a time range it could be serialized
32
+ # to the same value (and parsed back as well).
33
+ #
34
+ # if we don't know how to serialize something we provide a language-specific
35
+ # string of it and encode is at such.
36
+ #
37
+ # The output ought to use the UTF-8 encoding.
38
+ #
39
+ # This dumper has been inspired by the OkJSON gem (both formats look alike
40
+ # after all).
41
+ module Generator; extend self
42
+ SINGLE_QUOTE_MATCH = /'/
43
+ STRING_ESCAPE_MATCH = /[\s"=:{}\[\]]/
44
+
45
+ # max_nesting::
46
+ # After a certain depth, arrays are replaced with [...] and objects with
47
+ # {...}. Default is 4
48
+ # max_bytesize::
49
+ # After a certain lenght the root object is interrupted by the ...
50
+ # notation. Default is 4096.
51
+ def generate(obj, opts={}) #=> String
52
+ max_nesting = opts[:max_nesting] || 4
53
+ max_bytesize = opts[:max_bytesize] || 4096
54
+
55
+ depth = max_nesting - 1
56
+ bytesize = 0
57
+
58
+ line = obj.inject([]) do |acc, (k, v)|
59
+ if bytesize + (acc.size - 1) > max_bytesize
60
+ break acc
61
+ end
62
+ str = "#{keyenc(k)}=#{valenc(v, depth)}"
63
+ bytesize += str.bytesize
64
+ acc.push(str)
65
+ acc
66
+ end
67
+ if bytesize + (line.size - 1) > max_bytesize
68
+ if bytesize + (line.size - 1) - (line[-1].bytesize - 3) > max_bytesize
69
+ line.pop
70
+ end
71
+ line[-1] = DOT_DOT_DOT
72
+ end
73
+ line.join(SPACE)
74
+ end
75
+
76
+ protected
77
+
78
+ def valenc(x, depth)
79
+ case x
80
+ when Hash then objenc(x, depth)
81
+ when Array then arrenc(x, depth)
82
+ when String, Symbol then strenc(x)
83
+ when Numeric then numenc(x)
84
+ when Time then timeenc(x)
85
+ when true then LIT_TRUE
86
+ when false then LIT_FALSE
87
+ when nil then LIT_NIL
88
+ else
89
+ litenc(x)
90
+ end
91
+ end
92
+
93
+ def objenc_internal(x, depth)
94
+ depth -= 1
95
+ if depth < 0
96
+ DOT_DOT_DOT
97
+ else
98
+ x.map{|k,v| "#{keyenc(k)}=#{valenc(v, depth)}" }.join(SPACE)
99
+ end
100
+ end
101
+
102
+ def objenc(x, depth)
103
+ OPEN_BRACE + objenc_internal(x, depth) + SHUT_BRACE
104
+ end
105
+
106
+ def arrenc_internal(a, depth)
107
+ depth -= 1
108
+ if depth < 0
109
+ DOT_DOT_DOT
110
+ else
111
+ a.map{|x| valenc(x, depth)}.join(SPACE)
112
+ end
113
+ end
114
+
115
+ def arrenc(a, depth)
116
+ OPEN_BRACKET + arrenc_internal(a, depth) + SHUT_BRACKET
117
+ end
118
+
119
+ def keyenc(s)
120
+ s = s.to_s
121
+ # Poor-man's escaping
122
+ case s
123
+ when SINGLE_QUOTE_MATCH
124
+ s.inspect
125
+ when STRING_ESCAPE_MATCH
126
+ SINGLE_QUOTE +
127
+ s.inspect[1..-2].gsub(ESCAPED_DOUBLE_QUOTE, DOUBLE_QUOTE) +
128
+ SINGLE_QUOTE
129
+ else
130
+ s
131
+ end
132
+ end
133
+
134
+ def strenc(s)
135
+ s = s.to_s
136
+ # Poor-man's escaping
137
+ case s
138
+ when SINGLE_QUOTE_MATCH
139
+ s.inspect
140
+ when STRING_ESCAPE_MATCH, NUM_CAPTURE, LIT_TRUE, LIT_FALSE, LIT_NIL, DOT_DOT_DOT
141
+ SINGLE_QUOTE +
142
+ s.inspect[1..-2].gsub(ESCAPED_DOUBLE_QUOTE, DOUBLE_QUOTE) +
143
+ SINGLE_QUOTE
144
+ else
145
+ s
146
+ end
147
+ end
148
+
149
+ def numenc(n)
150
+ #case n
151
+ # when Float
152
+ # "%.3f" % n
153
+ #else
154
+ n.to_s
155
+ #end
156
+ end
157
+
158
+ def litenc(x)
159
+ strenc x.inspect
160
+ rescue
161
+ strenc (class << x; self; end).ancestors.first.inspect
162
+ end
163
+
164
+ def timeenc(t)
165
+ t.utc.iso8601
166
+ end
167
+ end
168
+ end
@@ -0,0 +1,182 @@
1
+ # -*- encoding : utf-8 -*-
2
+
3
+ require 'strscan'
4
+
5
+ require 'lines/common'
6
+
7
+ module Lines
8
+ class Parser
9
+ DOT_DOT_DOT_MATCH = /\.\.\./
10
+
11
+ LITERAL_MATCH = /[^=\s}\]]+/
12
+ SINGLE_QUOTE_MATCH = /(?:\\.|[^'])*/
13
+ DOUBLE_QUOTE_MATCH = /(?:\\.|[^"])*/
14
+
15
+ def self.parse(string, opts={})
16
+ new(string, opts).parse
17
+ end
18
+
19
+ def initialize(string, opts)
20
+ @s = StringScanner.new(string)
21
+ @opts = opts
22
+ end
23
+
24
+ def parse
25
+ inner_obj
26
+ end
27
+
28
+ protected
29
+
30
+ def accept(char)
31
+ if @s.peek(1) == char
32
+ @s.pos += 1
33
+ return true
34
+ end
35
+ false
36
+ end
37
+
38
+ def skip(num)
39
+ @s.pos += num
40
+ end
41
+
42
+ def expect(char)
43
+ if !accept(char)
44
+ fail "Expected '#{char}' but got '#{@s.peek(1)}'"
45
+ end
46
+ end
47
+
48
+ def fail(msg)
49
+ raise ParseError, "At #{@s}, #{msg}"
50
+ end
51
+
52
+ def dbg(*x)
53
+ #p [@s] + x
54
+ end
55
+
56
+ # Structures
57
+
58
+
59
+ def inner_obj
60
+ dbg :inner_obj
61
+ # Shortcut for the '...' max_depth notation
62
+ if @s.scan(DOT_DOT_DOT_MATCH)
63
+ return {DOT_DOT_DOT => ''}
64
+ end
65
+
66
+ return {} if @s.eos? || @s.peek(1) == SHUT_BRACE
67
+
68
+ # First pair
69
+ k = key()
70
+ expect EQUAL
71
+ obj = {
72
+ k => value()
73
+ }
74
+
75
+ while accept(SPACE) and !@s.eos?
76
+ k = key()
77
+ expect EQUAL
78
+ obj[k] = value()
79
+ end
80
+
81
+ obj
82
+ end
83
+
84
+ def key
85
+ dbg :key
86
+
87
+ ret = case @s.peek(1)
88
+ when SINGLE_QUOTE
89
+ single_quoted_string
90
+ when DOUBLE_QUOTE
91
+ double_quoted_string
92
+ else
93
+ literal(false)
94
+ end
95
+ @opts[:symbolize_names] ? ret.to_sym : ret
96
+ end
97
+
98
+ def single_quoted_string
99
+ dbg :single_quoted_string
100
+
101
+ expect SINGLE_QUOTE
102
+ str = @s.scan(SINGLE_QUOTE_MATCH).
103
+ gsub(ESCAPED_SINGLE_QUOTE, SINGLE_QUOTE)
104
+ expect SINGLE_QUOTE
105
+ str
106
+ end
107
+
108
+ def double_quoted_string
109
+ dbg :double_quoted_string
110
+
111
+ expect DOUBLE_QUOTE
112
+ str = @s.scan(DOUBLE_QUOTE_MATCH).
113
+ gsub(ESCAPED_DOUBLE_QUOTE, DOUBLE_QUOTE)
114
+ expect DOUBLE_QUOTE
115
+ str
116
+ end
117
+
118
+ def literal(sub_parse)
119
+ dbg :literal, sub_parse
120
+
121
+ literal = @s.scan LITERAL_MATCH
122
+
123
+ return "" unless literal
124
+
125
+ return literal unless sub_parse
126
+
127
+ case literal
128
+ when LIT_NIL
129
+ nil
130
+ when LIT_TRUE
131
+ true
132
+ when LIT_FALSE
133
+ false
134
+ when ISO8601_ZULU_CAPTURE
135
+ Time.new($1.to_i, $2.to_i, $3.to_i, $4.to_i, $5.to_i, $6.to_i, '+00:00').utc
136
+ when NUM_CAPTURE
137
+ literal.index('.') ? Float(literal) : Integer(literal)
138
+ else
139
+ literal
140
+ end
141
+ end
142
+
143
+ def value
144
+ dbg :value
145
+
146
+ case @s.peek(1)
147
+ when OPEN_BRACKET
148
+ list
149
+ when OPEN_BRACE
150
+ object
151
+ when DOUBLE_QUOTE
152
+ double_quoted_string
153
+ when SINGLE_QUOTE
154
+ single_quoted_string
155
+ else
156
+ literal(:sub_parse)
157
+ end
158
+ end
159
+
160
+ def list
161
+ dbg :list
162
+
163
+ list = []
164
+ expect(OPEN_BRACKET)
165
+ list.push value()
166
+ while accept(SPACE)
167
+ list.push value()
168
+ end
169
+ expect(SHUT_BRACKET)
170
+ list
171
+ end
172
+
173
+ def object
174
+ dbg :object
175
+
176
+ expect(OPEN_BRACE)
177
+ obj = inner_obj
178
+ expect(SHUT_BRACE)
179
+ obj
180
+ end
181
+ end
182
+ end
@@ -1,3 +1,3 @@
1
1
  module Lines
2
- VERSION = "0.2.0"
2
+ VERSION = "0.9.1"
3
3
  end
@@ -1,23 +1,25 @@
1
1
  # coding: utf-8
2
- lib = File.expand_path('../lib', __FILE__)
3
- $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
- require 'lines/version'
2
+ require File.expand_path('../lib/lines/version', __FILE__)
5
3
 
6
4
  Gem::Specification.new do |spec|
7
5
  spec.name = "lines"
8
6
  spec.version = Lines::VERSION
9
- spec.authors = ["Jonas Pfenniger"]
10
- spec.email = ["jonas@pfenniger.name"]
11
- spec.description = %q{structured logs for humans}
12
- spec.summary = %q{Lines is an opinionated structured logging library}
7
+ spec.authors = ["zimbatm"]
8
+ spec.email = ["zimbatm@zimbatm.com"]
9
+ spec.summary = %q{Lines is an opinionated structured log format}
10
+ spec.description = <<DESC
11
+ A log format that's readable by humans and easily parseable by computers.
12
+ DESC
13
13
  spec.homepage = 'https://github.com/zimbatm/lines-ruby'
14
14
  spec.license = "MIT"
15
15
 
16
- spec.files = `git ls-files`.split($/)
16
+ spec.files = `git ls-files .`.split($/)
17
17
  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
18
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
19
  spec.require_paths = ["lib"]
20
20
 
21
+ spec.add_development_dependency "benchmark-ips"
21
22
  spec.add_development_dependency "bundler", "~> 1.3"
23
+ spec.add_development_dependency "rake"
22
24
  spec.add_development_dependency "rspec"
23
25
  end
@@ -0,0 +1,45 @@
1
+ $:.unshift File.expand_path('../../lib', __FILE__)
2
+
3
+ require 'benchmark/ips'
4
+ require 'time'
5
+
6
+ $message = {
7
+ "at" => Time.now.utc.iso8601,
8
+ "pid" => Process.pid,
9
+ "app" => File.basename($0),
10
+ "pri" => "info",
11
+ "msg" => "This is my message",
12
+ "user" => {"t" => true, "f" => false, "n" => nil},
13
+ "elapsed" => [55.67, 'ms'],
14
+ }
15
+
16
+ formatters = [
17
+ ['lines', "Lines.dump($message)"],
18
+
19
+ ['json/pure', "JSON.dump($message)"],
20
+ ['oj', "Oj.dump($message)"],
21
+ ['yajl', "Yajl.dump($message)"],
22
+
23
+ ['msgpack', "MessagePack.dump($message)"],
24
+ ['bson', "$message.to_bson"],
25
+ ['tnetstring', "TNetstring.dump($message)"],
26
+ ]
27
+
28
+ puts "%-12s %-5s %s" % ['format', 'size', 'output']
29
+ puts "-" * 25
30
+
31
+ Benchmark.ips do |x|
32
+ x.compare!
33
+ formatters.each do |(feature, action)|
34
+ begin
35
+ require feature
36
+
37
+ data = eval action
38
+ puts "%-12s %-5d %s" % [feature, data.size, data]
39
+
40
+ x.report feature, action
41
+ rescue LoadError
42
+ puts "%-12s could not be loaded" % [feature]
43
+ end
44
+ end
45
+ end