lines 0.2.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +6 -14
- data/.gitignore +1 -0
- data/.travis.yml +1 -1
- data/CHANGELOG.md +16 -0
- data/Gemfile +1 -8
- data/LICENSE.txt +1 -1
- data/README.md +21 -64
- data/examples/cli.rb +17 -0
- data/lib/lines.rb +111 -395
- data/lib/lines/common.rb +37 -0
- data/lib/lines/generator.rb +168 -0
- data/lib/lines/parser.rb +182 -0
- data/lib/lines/version.rb +1 -1
- data/lines.gemspec +10 -8
- data/spec/lines_generator_bench.rb +45 -0
- data/spec/lines_generator_spec.rb +65 -0
- data/spec/lines_parser_bench.rb +50 -0
- data/spec/{lines_loader_spec.rb → lines_parser_spec.rb} +28 -7
- data/spec/spec_helper.rb +2 -6
- metadata +57 -28
- data/lib/lines/active_record.rb +0 -70
- data/lib/lines/loader.rb +0 -229
- data/lib/lines/logger.rb +0 -61
- data/lib/lines/rack_logger.rb +0 -39
- data/spec/bench.rb +0 -46
- data/spec/lines_spec.rb +0 -190
- data/spec/parse-bench +0 -26
data/lib/lines/common.rb
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
# -*- encoding : utf-8 -*-
|
2
|
+
|
3
|
+
module Lines
|
4
|
+
module Error
|
5
|
+
# Used to mark non-lines errors as being part of the library. This lets
|
6
|
+
# a library user `rescue Lines::Error => ex` and catch all exceptions
|
7
|
+
# comming from the lines library.
|
8
|
+
def self.tag(obj)
|
9
|
+
obj.extend Error
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class ParseError < StandardError; include Error; end
|
14
|
+
#class LogicError < RuntimeError; include Error; end
|
15
|
+
|
16
|
+
LIT_TRUE = '#t'
|
17
|
+
LIT_FALSE = '#f'
|
18
|
+
LIT_NIL = 'nil'
|
19
|
+
|
20
|
+
SPACE = ' '
|
21
|
+
EQUAL = '='
|
22
|
+
OPEN_BRACE = '{'
|
23
|
+
SHUT_BRACE = '}'
|
24
|
+
OPEN_BRACKET = '['
|
25
|
+
SHUT_BRACKET = ']'
|
26
|
+
SINGLE_QUOTE = "'"
|
27
|
+
DOUBLE_QUOTE = '"'
|
28
|
+
DOT_DOT_DOT = '...'
|
29
|
+
|
30
|
+
BACKSLASH = '\\'
|
31
|
+
ESCAPED_SINGLE_QUOTE = "\\'"
|
32
|
+
ESCAPED_DOUBLE_QUOTE = '\"'
|
33
|
+
|
34
|
+
NUM_MATCH = /-?(?:0|[1-9])\d*(?:\.\d+)?(?:[eE][+-]\d+)?/
|
35
|
+
ISO8601_ZULU_CAPTURE = /^(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)Z$/
|
36
|
+
NUM_CAPTURE = /^(#{NUM_MATCH})$/
|
37
|
+
end
|
@@ -0,0 +1,168 @@
|
|
1
|
+
# -*- encoding : utf-8 -*-
|
2
|
+
|
3
|
+
require 'time'
|
4
|
+
|
5
|
+
require 'lines/common'
|
6
|
+
|
7
|
+
module Lines
|
8
|
+
# Some opinions here as well on the format:
|
9
|
+
#
|
10
|
+
# We really want to never fail at dumping because you know, they're logs.
|
11
|
+
# It's better to get a slightly less readable log that no logs at all.
|
12
|
+
#
|
13
|
+
# We're trying to be helpful for humans. It means that if possible we want
|
14
|
+
# to make things shorter and more readable. It also means that ideally
|
15
|
+
# we would like the parsing to be isomorphic but approximations are alright.
|
16
|
+
# For example a symbol might become a string.
|
17
|
+
#
|
18
|
+
# Basically, values are either composite (dictionaries and arrays), quoted
|
19
|
+
# strings or litterals. Litterals are strings that can be parsed to
|
20
|
+
# something else depending if the language supports it or not.
|
21
|
+
# Litterals never contain white-spaces or other weird (very precise !) characters.
|
22
|
+
#
|
23
|
+
# the true litteral is written as "#t"
|
24
|
+
# the false litteral is written as "#f"
|
25
|
+
# the nil / null litteral is written as "nil"
|
26
|
+
#
|
27
|
+
# dictionary keys are always strings or litterals.
|
28
|
+
#
|
29
|
+
# Pleaaase, keep units with numbers. And we provide a way for this:
|
30
|
+
# a tuple of (number, litteral) can be concatenated. Eg: (3, 'ms') => 3:ms
|
31
|
+
# alternatively if your language supports a time range it could be serialized
|
32
|
+
# to the same value (and parsed back as well).
|
33
|
+
#
|
34
|
+
# if we don't know how to serialize something we provide a language-specific
|
35
|
+
# string of it and encode is at such.
|
36
|
+
#
|
37
|
+
# The output ought to use the UTF-8 encoding.
|
38
|
+
#
|
39
|
+
# This dumper has been inspired by the OkJSON gem (both formats look alike
|
40
|
+
# after all).
|
41
|
+
module Generator; extend self
|
42
|
+
SINGLE_QUOTE_MATCH = /'/
|
43
|
+
STRING_ESCAPE_MATCH = /[\s"=:{}\[\]]/
|
44
|
+
|
45
|
+
# max_nesting::
|
46
|
+
# After a certain depth, arrays are replaced with [...] and objects with
|
47
|
+
# {...}. Default is 4
|
48
|
+
# max_bytesize::
|
49
|
+
# After a certain lenght the root object is interrupted by the ...
|
50
|
+
# notation. Default is 4096.
|
51
|
+
def generate(obj, opts={}) #=> String
|
52
|
+
max_nesting = opts[:max_nesting] || 4
|
53
|
+
max_bytesize = opts[:max_bytesize] || 4096
|
54
|
+
|
55
|
+
depth = max_nesting - 1
|
56
|
+
bytesize = 0
|
57
|
+
|
58
|
+
line = obj.inject([]) do |acc, (k, v)|
|
59
|
+
if bytesize + (acc.size - 1) > max_bytesize
|
60
|
+
break acc
|
61
|
+
end
|
62
|
+
str = "#{keyenc(k)}=#{valenc(v, depth)}"
|
63
|
+
bytesize += str.bytesize
|
64
|
+
acc.push(str)
|
65
|
+
acc
|
66
|
+
end
|
67
|
+
if bytesize + (line.size - 1) > max_bytesize
|
68
|
+
if bytesize + (line.size - 1) - (line[-1].bytesize - 3) > max_bytesize
|
69
|
+
line.pop
|
70
|
+
end
|
71
|
+
line[-1] = DOT_DOT_DOT
|
72
|
+
end
|
73
|
+
line.join(SPACE)
|
74
|
+
end
|
75
|
+
|
76
|
+
protected
|
77
|
+
|
78
|
+
def valenc(x, depth)
|
79
|
+
case x
|
80
|
+
when Hash then objenc(x, depth)
|
81
|
+
when Array then arrenc(x, depth)
|
82
|
+
when String, Symbol then strenc(x)
|
83
|
+
when Numeric then numenc(x)
|
84
|
+
when Time then timeenc(x)
|
85
|
+
when true then LIT_TRUE
|
86
|
+
when false then LIT_FALSE
|
87
|
+
when nil then LIT_NIL
|
88
|
+
else
|
89
|
+
litenc(x)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def objenc_internal(x, depth)
|
94
|
+
depth -= 1
|
95
|
+
if depth < 0
|
96
|
+
DOT_DOT_DOT
|
97
|
+
else
|
98
|
+
x.map{|k,v| "#{keyenc(k)}=#{valenc(v, depth)}" }.join(SPACE)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def objenc(x, depth)
|
103
|
+
OPEN_BRACE + objenc_internal(x, depth) + SHUT_BRACE
|
104
|
+
end
|
105
|
+
|
106
|
+
def arrenc_internal(a, depth)
|
107
|
+
depth -= 1
|
108
|
+
if depth < 0
|
109
|
+
DOT_DOT_DOT
|
110
|
+
else
|
111
|
+
a.map{|x| valenc(x, depth)}.join(SPACE)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def arrenc(a, depth)
|
116
|
+
OPEN_BRACKET + arrenc_internal(a, depth) + SHUT_BRACKET
|
117
|
+
end
|
118
|
+
|
119
|
+
def keyenc(s)
|
120
|
+
s = s.to_s
|
121
|
+
# Poor-man's escaping
|
122
|
+
case s
|
123
|
+
when SINGLE_QUOTE_MATCH
|
124
|
+
s.inspect
|
125
|
+
when STRING_ESCAPE_MATCH
|
126
|
+
SINGLE_QUOTE +
|
127
|
+
s.inspect[1..-2].gsub(ESCAPED_DOUBLE_QUOTE, DOUBLE_QUOTE) +
|
128
|
+
SINGLE_QUOTE
|
129
|
+
else
|
130
|
+
s
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
def strenc(s)
|
135
|
+
s = s.to_s
|
136
|
+
# Poor-man's escaping
|
137
|
+
case s
|
138
|
+
when SINGLE_QUOTE_MATCH
|
139
|
+
s.inspect
|
140
|
+
when STRING_ESCAPE_MATCH, NUM_CAPTURE, LIT_TRUE, LIT_FALSE, LIT_NIL, DOT_DOT_DOT
|
141
|
+
SINGLE_QUOTE +
|
142
|
+
s.inspect[1..-2].gsub(ESCAPED_DOUBLE_QUOTE, DOUBLE_QUOTE) +
|
143
|
+
SINGLE_QUOTE
|
144
|
+
else
|
145
|
+
s
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
def numenc(n)
|
150
|
+
#case n
|
151
|
+
# when Float
|
152
|
+
# "%.3f" % n
|
153
|
+
#else
|
154
|
+
n.to_s
|
155
|
+
#end
|
156
|
+
end
|
157
|
+
|
158
|
+
def litenc(x)
|
159
|
+
strenc x.inspect
|
160
|
+
rescue
|
161
|
+
strenc (class << x; self; end).ancestors.first.inspect
|
162
|
+
end
|
163
|
+
|
164
|
+
def timeenc(t)
|
165
|
+
t.utc.iso8601
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
data/lib/lines/parser.rb
ADDED
@@ -0,0 +1,182 @@
|
|
1
|
+
# -*- encoding : utf-8 -*-
|
2
|
+
|
3
|
+
require 'strscan'
|
4
|
+
|
5
|
+
require 'lines/common'
|
6
|
+
|
7
|
+
module Lines
|
8
|
+
class Parser
|
9
|
+
DOT_DOT_DOT_MATCH = /\.\.\./
|
10
|
+
|
11
|
+
LITERAL_MATCH = /[^=\s}\]]+/
|
12
|
+
SINGLE_QUOTE_MATCH = /(?:\\.|[^'])*/
|
13
|
+
DOUBLE_QUOTE_MATCH = /(?:\\.|[^"])*/
|
14
|
+
|
15
|
+
def self.parse(string, opts={})
|
16
|
+
new(string, opts).parse
|
17
|
+
end
|
18
|
+
|
19
|
+
def initialize(string, opts)
|
20
|
+
@s = StringScanner.new(string)
|
21
|
+
@opts = opts
|
22
|
+
end
|
23
|
+
|
24
|
+
def parse
|
25
|
+
inner_obj
|
26
|
+
end
|
27
|
+
|
28
|
+
protected
|
29
|
+
|
30
|
+
def accept(char)
|
31
|
+
if @s.peek(1) == char
|
32
|
+
@s.pos += 1
|
33
|
+
return true
|
34
|
+
end
|
35
|
+
false
|
36
|
+
end
|
37
|
+
|
38
|
+
def skip(num)
|
39
|
+
@s.pos += num
|
40
|
+
end
|
41
|
+
|
42
|
+
def expect(char)
|
43
|
+
if !accept(char)
|
44
|
+
fail "Expected '#{char}' but got '#{@s.peek(1)}'"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def fail(msg)
|
49
|
+
raise ParseError, "At #{@s}, #{msg}"
|
50
|
+
end
|
51
|
+
|
52
|
+
def dbg(*x)
|
53
|
+
#p [@s] + x
|
54
|
+
end
|
55
|
+
|
56
|
+
# Structures
|
57
|
+
|
58
|
+
|
59
|
+
def inner_obj
|
60
|
+
dbg :inner_obj
|
61
|
+
# Shortcut for the '...' max_depth notation
|
62
|
+
if @s.scan(DOT_DOT_DOT_MATCH)
|
63
|
+
return {DOT_DOT_DOT => ''}
|
64
|
+
end
|
65
|
+
|
66
|
+
return {} if @s.eos? || @s.peek(1) == SHUT_BRACE
|
67
|
+
|
68
|
+
# First pair
|
69
|
+
k = key()
|
70
|
+
expect EQUAL
|
71
|
+
obj = {
|
72
|
+
k => value()
|
73
|
+
}
|
74
|
+
|
75
|
+
while accept(SPACE) and !@s.eos?
|
76
|
+
k = key()
|
77
|
+
expect EQUAL
|
78
|
+
obj[k] = value()
|
79
|
+
end
|
80
|
+
|
81
|
+
obj
|
82
|
+
end
|
83
|
+
|
84
|
+
def key
|
85
|
+
dbg :key
|
86
|
+
|
87
|
+
ret = case @s.peek(1)
|
88
|
+
when SINGLE_QUOTE
|
89
|
+
single_quoted_string
|
90
|
+
when DOUBLE_QUOTE
|
91
|
+
double_quoted_string
|
92
|
+
else
|
93
|
+
literal(false)
|
94
|
+
end
|
95
|
+
@opts[:symbolize_names] ? ret.to_sym : ret
|
96
|
+
end
|
97
|
+
|
98
|
+
def single_quoted_string
|
99
|
+
dbg :single_quoted_string
|
100
|
+
|
101
|
+
expect SINGLE_QUOTE
|
102
|
+
str = @s.scan(SINGLE_QUOTE_MATCH).
|
103
|
+
gsub(ESCAPED_SINGLE_QUOTE, SINGLE_QUOTE)
|
104
|
+
expect SINGLE_QUOTE
|
105
|
+
str
|
106
|
+
end
|
107
|
+
|
108
|
+
def double_quoted_string
|
109
|
+
dbg :double_quoted_string
|
110
|
+
|
111
|
+
expect DOUBLE_QUOTE
|
112
|
+
str = @s.scan(DOUBLE_QUOTE_MATCH).
|
113
|
+
gsub(ESCAPED_DOUBLE_QUOTE, DOUBLE_QUOTE)
|
114
|
+
expect DOUBLE_QUOTE
|
115
|
+
str
|
116
|
+
end
|
117
|
+
|
118
|
+
def literal(sub_parse)
|
119
|
+
dbg :literal, sub_parse
|
120
|
+
|
121
|
+
literal = @s.scan LITERAL_MATCH
|
122
|
+
|
123
|
+
return "" unless literal
|
124
|
+
|
125
|
+
return literal unless sub_parse
|
126
|
+
|
127
|
+
case literal
|
128
|
+
when LIT_NIL
|
129
|
+
nil
|
130
|
+
when LIT_TRUE
|
131
|
+
true
|
132
|
+
when LIT_FALSE
|
133
|
+
false
|
134
|
+
when ISO8601_ZULU_CAPTURE
|
135
|
+
Time.new($1.to_i, $2.to_i, $3.to_i, $4.to_i, $5.to_i, $6.to_i, '+00:00').utc
|
136
|
+
when NUM_CAPTURE
|
137
|
+
literal.index('.') ? Float(literal) : Integer(literal)
|
138
|
+
else
|
139
|
+
literal
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
def value
|
144
|
+
dbg :value
|
145
|
+
|
146
|
+
case @s.peek(1)
|
147
|
+
when OPEN_BRACKET
|
148
|
+
list
|
149
|
+
when OPEN_BRACE
|
150
|
+
object
|
151
|
+
when DOUBLE_QUOTE
|
152
|
+
double_quoted_string
|
153
|
+
when SINGLE_QUOTE
|
154
|
+
single_quoted_string
|
155
|
+
else
|
156
|
+
literal(:sub_parse)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def list
|
161
|
+
dbg :list
|
162
|
+
|
163
|
+
list = []
|
164
|
+
expect(OPEN_BRACKET)
|
165
|
+
list.push value()
|
166
|
+
while accept(SPACE)
|
167
|
+
list.push value()
|
168
|
+
end
|
169
|
+
expect(SHUT_BRACKET)
|
170
|
+
list
|
171
|
+
end
|
172
|
+
|
173
|
+
def object
|
174
|
+
dbg :object
|
175
|
+
|
176
|
+
expect(OPEN_BRACE)
|
177
|
+
obj = inner_obj
|
178
|
+
expect(SHUT_BRACE)
|
179
|
+
obj
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
data/lib/lines/version.rb
CHANGED
data/lines.gemspec
CHANGED
@@ -1,23 +1,25 @@
|
|
1
1
|
# coding: utf-8
|
2
|
-
|
3
|
-
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require 'lines/version'
|
2
|
+
require File.expand_path('../lib/lines/version', __FILE__)
|
5
3
|
|
6
4
|
Gem::Specification.new do |spec|
|
7
5
|
spec.name = "lines"
|
8
6
|
spec.version = Lines::VERSION
|
9
|
-
spec.authors = ["
|
10
|
-
spec.email = ["
|
11
|
-
spec.
|
12
|
-
spec.
|
7
|
+
spec.authors = ["zimbatm"]
|
8
|
+
spec.email = ["zimbatm@zimbatm.com"]
|
9
|
+
spec.summary = %q{Lines is an opinionated structured log format}
|
10
|
+
spec.description = <<DESC
|
11
|
+
A log format that's readable by humans and easily parseable by computers.
|
12
|
+
DESC
|
13
13
|
spec.homepage = 'https://github.com/zimbatm/lines-ruby'
|
14
14
|
spec.license = "MIT"
|
15
15
|
|
16
|
-
spec.files = `git ls-files
|
16
|
+
spec.files = `git ls-files .`.split($/)
|
17
17
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
18
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
19
|
spec.require_paths = ["lib"]
|
20
20
|
|
21
|
+
spec.add_development_dependency "benchmark-ips"
|
21
22
|
spec.add_development_dependency "bundler", "~> 1.3"
|
23
|
+
spec.add_development_dependency "rake"
|
22
24
|
spec.add_development_dependency "rspec"
|
23
25
|
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
$:.unshift File.expand_path('../../lib', __FILE__)
|
2
|
+
|
3
|
+
require 'benchmark/ips'
|
4
|
+
require 'time'
|
5
|
+
|
6
|
+
$message = {
|
7
|
+
"at" => Time.now.utc.iso8601,
|
8
|
+
"pid" => Process.pid,
|
9
|
+
"app" => File.basename($0),
|
10
|
+
"pri" => "info",
|
11
|
+
"msg" => "This is my message",
|
12
|
+
"user" => {"t" => true, "f" => false, "n" => nil},
|
13
|
+
"elapsed" => [55.67, 'ms'],
|
14
|
+
}
|
15
|
+
|
16
|
+
formatters = [
|
17
|
+
['lines', "Lines.dump($message)"],
|
18
|
+
|
19
|
+
['json/pure', "JSON.dump($message)"],
|
20
|
+
['oj', "Oj.dump($message)"],
|
21
|
+
['yajl', "Yajl.dump($message)"],
|
22
|
+
|
23
|
+
['msgpack', "MessagePack.dump($message)"],
|
24
|
+
['bson', "$message.to_bson"],
|
25
|
+
['tnetstring', "TNetstring.dump($message)"],
|
26
|
+
]
|
27
|
+
|
28
|
+
puts "%-12s %-5s %s" % ['format', 'size', 'output']
|
29
|
+
puts "-" * 25
|
30
|
+
|
31
|
+
Benchmark.ips do |x|
|
32
|
+
x.compare!
|
33
|
+
formatters.each do |(feature, action)|
|
34
|
+
begin
|
35
|
+
require feature
|
36
|
+
|
37
|
+
data = eval action
|
38
|
+
puts "%-12s %-5d %s" % [feature, data.size, data]
|
39
|
+
|
40
|
+
x.report feature, action
|
41
|
+
rescue LoadError
|
42
|
+
puts "%-12s could not be loaded" % [feature]
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|