lines 0.2.0 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,65 @@
1
+ require 'spec_helper'
2
+ require 'lines/generator'
3
+
4
+ describe Lines::Generator do
5
+ def expect_dump(obj, opts={})
6
+ expect(subject.generate obj, opts)
7
+ end
8
+
9
+ it "can dump stuff" do
10
+ expect_dump(foo: "bar", bar: 33).to eq('foo=bar bar=33')
11
+ end
12
+
13
+ it "handles max_depth items" do
14
+ expect_dump({x: [444]}, max_nesting: 1).to eq('x=[...]')
15
+ expect_dump({x: {y: 444}}, max_nesting: 1).to eq('x={...}')
16
+ end
17
+
18
+ it "handles max_size items" do
19
+ expect_dump({a: "aaa", b: "bbb", c: "cccc"}, max_bytesize: 18).to eq('a=aaa b=bbb c=cccc')
20
+ expect_dump({a: "aaa", b: "bbb", c: "cccc"}, max_bytesize: 17).to eq('a=aaa b=bbb ...')
21
+ expect_dump({a: "aaa", b: "bbb", c: "cccc"}, max_bytesize: 15).to eq('a=aaa b=bbb ...')
22
+ expect_dump({a: "aaa", b: "bbb", c: "cccc"}, max_bytesize: 14).to eq('a=aaa ...')
23
+ end
24
+
25
+ it "treats missing value in a pair as an empty string" do
26
+ expect_dump(x: '').to eq("x=")
27
+ end
28
+
29
+ it "escapes quotes in quoted strings" do
30
+ expect_dump(x: "foo'bar").to eq("x=\"foo'bar\"")
31
+ expect_dump(x: 'foo"bar').to eq("x='foo\"bar'")
32
+ end
33
+
34
+ it "doesn't parse literals when they are keys" do
35
+ expect_dump(3 => 4).to eq("3=4")
36
+ end
37
+
38
+ it "escapes ambiguous strings" do
39
+ expect_dump(x: '4', y: '-3.3').to eq("x='4' y='-3.3'")
40
+ expect_dump(a: '#t', b: '#f', c: 'nil').to eq("a='#t' b='#f' c='nil'")
41
+ end
42
+
43
+ it "handles some random stuff" do
44
+ expect_dump("" => "").to eq("=")
45
+ expect_dump('"' => 'zzz').to eq('\'"\'=zzz')
46
+ end
47
+
48
+ it "escapes various whitespace characters" do
49
+ expect_dump("\r\n\t" => "\r\n\t").to eq('\'\r\n\t\'=\'\r\n\t\'')
50
+ end
51
+
52
+ it "generates sample log lines" do
53
+ expect_dump("commit" => "716f337").to eq("commit=716f337")
54
+
55
+ line = <<LINE.rstrip
56
+ at=2013-07-12T21:33:47Z commit=716f337 sql="SELECT FROM_UNIXTIME(UNIX_TIMESTAMP(created_at) - UNIX_TIMESTAMP(created_at)%(300)) as timestamp FROM `job_queue_logs` WHERE `job_queue_logs`.`account_id` = 'effe376baf553c590c02090abe512278' AND (created_at >= '2013-06-28 16:56:12') GROUP BY timestamp" elapsed=[31.9 ms]
57
+ LINE
58
+ expect_dump(
59
+ "at" => Time.at(1373664827).utc,
60
+ "commit" => "716f337",
61
+ "sql" => "SELECT FROM_UNIXTIME(UNIX_TIMESTAMP(created_at) - UNIX_TIMESTAMP(created_at)%(300)) as timestamp FROM `job_queue_logs` WHERE `job_queue_logs`.`account_id` = 'effe376baf553c590c02090abe512278' AND (created_at >= '2013-06-28 16:56:12') GROUP BY timestamp",
62
+ "elapsed" => [31.9, "ms"],
63
+ ).to eq(line)
64
+ end
65
+ end
@@ -0,0 +1,50 @@
1
+ $:.unshift File.expand_path('../../lib', __FILE__)
2
+
3
+ require 'benchmark/ips'
4
+ require 'time'
5
+
6
+ $message = {
7
+ "at" => Time.now.utc.iso8601,
8
+ "pid" => Process.pid,
9
+ "app" => File.basename($0),
10
+ "pri" => "info",
11
+ "msg" => "This is my message",
12
+ "user" => {"t" => true, "f" => false, "n" => nil},
13
+ "elapsed" => [55.67, 'ms'],
14
+ }
15
+
16
+ $data = {}
17
+
18
+ formatters = [
19
+ ['lines', "Lines.dump($message)", "Lines.load($data[:lines])"],
20
+
21
+ ['json/pure', "JSON.dump($message)", "JSON.load($data[:'json/pure'])"],
22
+ ['oj', "Oj.dump($message)", "Oj.load($data[:oj])"],
23
+ ['yajl', "Yajl.dump($message)", "Yajl.load($data[:yajl])"],
24
+
25
+ ['msgpack', "MessagePack.dump($message)", "MessagePack.load($data[:msgpack])"],
26
+ ['bson', "$message.to_bson", "???"],
27
+ ['tnetstring', "TNetstring.dump($message)", "TNetstring.load($data[:tnetstring])"],
28
+ ]
29
+
30
+ Benchmark.ips do |x|
31
+ x.compare!
32
+ formatters.each do |(feature, dumper, loader)|
33
+ begin
34
+ require feature
35
+
36
+ $data[feature.to_sym] = eval dumper
37
+ #puts "%-12s %-5d %s" % [feature, data.size, data]
38
+
39
+ msg = eval loader
40
+
41
+ if $message != msg
42
+ p [:invalid, $message, msg]
43
+ end
44
+
45
+ x.report feature, loader
46
+ rescue LoadError
47
+ puts "%-12s could not be loaded" % [feature]
48
+ end
49
+ end
50
+ end
@@ -1,11 +1,11 @@
1
1
  require 'spec_helper'
2
- require 'lines/loader'
2
+ require 'lines/parser'
3
3
 
4
- describe Lines::Loader do
5
- subject { Lines::Loader.new }
4
+ describe Lines::Parser do
5
+ subject{ Lines::Parser }
6
6
 
7
- def expect_load(str)
8
- expect(subject.parse str)
7
+ def expect_load(str, opts={})
8
+ expect(subject.parse str, opts)
9
9
  end
10
10
 
11
11
  it "can load stuff" do
@@ -17,12 +17,20 @@ describe Lines::Loader do
17
17
  expect_load('x={...}').to eq("x" => {"..." => ""})
18
18
  end
19
19
 
20
+ it "handles unfinished lines" do
21
+ expect_load('x=foo ').to eq("x" => "foo")
22
+ end
23
+
24
+ it "handles \\n at the end of line" do
25
+ expect_load("x=foo\n").to eq("x" => "foo")
26
+ end
27
+
20
28
  it "treats missing value in a pair as an empty string" do
21
29
  expect_load('x=').to eq("x" => "")
22
30
  end
23
31
 
24
32
  it "has non-greedy string parsing" do
25
- expect_load('x="foo" bar="baz"').to eq("x" => "foo", "bar" => "baz")
33
+ expect_load('x="#t" bar="baz"').to eq("x" => "#t", "bar" => "baz")
26
34
  end
27
35
 
28
36
  it "unscapes quotes in quoted strings" do
@@ -39,11 +47,24 @@ describe Lines::Loader do
39
47
  expect_load('"\""=zzz').to eq('"' => "zzz")
40
48
  end
41
49
 
50
+ it "knows how to restore iso time" do
51
+ expect_load("at=2013-07-12T21:33:47Z").to eq("at" => Time.at(1373664827).utc)
52
+ end
53
+
54
+ it "can symbolize the names" do
55
+ expect_load("foo=bar", symbolize_names: true).to eq(foo: "bar")
56
+ end
57
+
58
+ it "restores escaped characters from a string" do
59
+ pending
60
+ expect_load('\'\r\n\t\'=\'\r\n\t\'').to eq("\r\n\t" => "\r\n\t")
61
+ end
62
+
42
63
  it "parses sample log lines" do
43
64
  expect_load("commit=716f337").to eq("commit" => "716f337")
44
65
 
45
66
  line = <<LINE
46
- at=2013-07-12T21:33:47Z commit=716f337 sql="SELECT FROM_UNIXTIME(UNIX_TIMESTAMP(created_at) - UNIX_TIMESTAMP(created_at)%(300)) as timestamp FROM `job_queue_logs` WHERE `job_queue_logs`.`account_id` = 'effe376baf553c590c02090abe512278' AND (created_at >= '2013-06-28 16:56:12') GROUP BY timestamp" elapsed=31.9:ms
67
+ at=2013-07-12T21:33:47Z commit=716f337 sql="SELECT FROM_UNIXTIME(UNIX_TIMESTAMP(created_at) - UNIX_TIMESTAMP(created_at)%(300)) as timestamp FROM `job_queue_logs` WHERE `job_queue_logs`.`account_id` = 'effe376baf553c590c02090abe512278' AND (created_at >= '2013-06-28 16:56:12') GROUP BY timestamp" elapsed=[31.9 ms]
47
68
  LINE
48
69
  expect_load(line).to eq(
49
70
  "at" => Time.at(1373664827).utc,
@@ -1,17 +1,13 @@
1
1
  # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
2
2
  RSpec.configure do |config|
3
- config.treat_symbols_as_metadata_keys_with_true_values = true
4
- config.run_all_when_everything_filtered = true
5
- config.filter_run :focus
6
-
3
+ # Only use the expect syntax
7
4
  config.expect_with :rspec do |c|
8
5
  c.syntax = :expect
9
6
  end
10
7
 
11
-
12
8
  # Run specs in random order to surface order dependencies. If you find an
13
9
  # order dependency and want to debug it, you can fix the order by providing
14
10
  # the seed, which is printed after each run.
15
11
  # --seed 1234
16
12
  config.order = 'random'
17
- end
13
+ end
metadata CHANGED
@@ -1,69 +1,98 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: lines
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.9.1
5
5
  platform: ruby
6
6
  authors:
7
- - Jonas Pfenniger
7
+ - zimbatm
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2013-07-15 00:00:00.000000000 Z
11
+ date: 2014-11-12 00:00:00.000000000 Z
12
12
  dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: benchmark-ips
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
13
27
  - !ruby/object:Gem::Dependency
14
28
  name: bundler
15
29
  requirement: !ruby/object:Gem::Requirement
16
30
  requirements:
17
- - - ~>
31
+ - - "~>"
18
32
  - !ruby/object:Gem::Version
19
33
  version: '1.3'
20
34
  type: :development
21
35
  prerelease: false
22
36
  version_requirements: !ruby/object:Gem::Requirement
23
37
  requirements:
24
- - - ~>
38
+ - - "~>"
25
39
  - !ruby/object:Gem::Version
26
40
  version: '1.3'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
27
55
  - !ruby/object:Gem::Dependency
28
56
  name: rspec
29
57
  requirement: !ruby/object:Gem::Requirement
30
58
  requirements:
31
- - - ! '>='
59
+ - - ">="
32
60
  - !ruby/object:Gem::Version
33
61
  version: '0'
34
62
  type: :development
35
63
  prerelease: false
36
64
  version_requirements: !ruby/object:Gem::Requirement
37
65
  requirements:
38
- - - ! '>='
66
+ - - ">="
39
67
  - !ruby/object:Gem::Version
40
68
  version: '0'
41
- description: structured logs for humans
69
+ description: |
70
+ A log format that's readable by humans and easily parseable by computers.
42
71
  email:
43
- - jonas@pfenniger.name
72
+ - zimbatm@zimbatm.com
44
73
  executables: []
45
74
  extensions: []
46
75
  extra_rdoc_files: []
47
76
  files:
48
- - .gitignore
49
- - .rspec
50
- - .travis.yml
77
+ - ".gitignore"
78
+ - ".rspec"
79
+ - ".travis.yml"
51
80
  - CHANGELOG.md
52
81
  - Gemfile
53
82
  - LICENSE.txt
54
83
  - README.md
55
84
  - Rakefile
85
+ - examples/cli.rb
56
86
  - lib/lines.rb
57
- - lib/lines/active_record.rb
58
- - lib/lines/loader.rb
59
- - lib/lines/logger.rb
60
- - lib/lines/rack_logger.rb
87
+ - lib/lines/common.rb
88
+ - lib/lines/generator.rb
89
+ - lib/lines/parser.rb
61
90
  - lib/lines/version.rb
62
91
  - lines.gemspec
63
- - spec/bench.rb
64
- - spec/lines_loader_spec.rb
65
- - spec/lines_spec.rb
66
- - spec/parse-bench
92
+ - spec/lines_generator_bench.rb
93
+ - spec/lines_generator_spec.rb
94
+ - spec/lines_parser_bench.rb
95
+ - spec/lines_parser_spec.rb
67
96
  - spec/spec_helper.rb
68
97
  homepage: https://github.com/zimbatm/lines-ruby
69
98
  licenses:
@@ -75,23 +104,23 @@ require_paths:
75
104
  - lib
76
105
  required_ruby_version: !ruby/object:Gem::Requirement
77
106
  requirements:
78
- - - ! '>='
107
+ - - ">="
79
108
  - !ruby/object:Gem::Version
80
109
  version: '0'
81
110
  required_rubygems_version: !ruby/object:Gem::Requirement
82
111
  requirements:
83
- - - ! '>='
112
+ - - ">="
84
113
  - !ruby/object:Gem::Version
85
114
  version: '0'
86
115
  requirements: []
87
116
  rubyforge_project:
88
- rubygems_version: 2.0.3
117
+ rubygems_version: 2.4.2
89
118
  signing_key:
90
119
  specification_version: 4
91
- summary: Lines is an opinionated structured logging library
120
+ summary: Lines is an opinionated structured log format
92
121
  test_files:
93
- - spec/bench.rb
94
- - spec/lines_loader_spec.rb
95
- - spec/lines_spec.rb
96
- - spec/parse-bench
122
+ - spec/lines_generator_bench.rb
123
+ - spec/lines_generator_spec.rb
124
+ - spec/lines_parser_bench.rb
125
+ - spec/lines_parser_spec.rb
97
126
  - spec/spec_helper.rb
@@ -1,70 +0,0 @@
1
- require 'active_record'
2
- require 'active_record/log_subscriber'
3
- require 'lines'
4
-
5
- module Lines
6
- class ActiveRecordSubscriber < ActiveSupport::LogSubscriber
7
- def render_bind(column, value)
8
- if column
9
- if column.binary?
10
- value = "<#{value.bytesize} bytes of binary data>"
11
- end
12
-
13
- [column.name, value]
14
- else
15
- [nil, value]
16
- end
17
- end
18
-
19
- def sql(event)
20
- payload = event.payload
21
-
22
- return if payload[:name] == 'SCHEMA' || payload[:name] == 'EXPLAIN'
23
-
24
- args = {}
25
-
26
- args[:name] = payload[:name] if payload[:name]
27
- args[:sql] = payload[:sql].squeeze(' ')
28
-
29
- if payload[:binds] && payload[:binds].any?
30
- args[:binds] = payload[:binds].inject({}) do |hash,(col, v)|
31
- k, v = render_bind(col, v)
32
- hash[k] = v
33
- hash
34
- end
35
- end
36
-
37
- args[:elapsed] = [event.duration.round(1), 'ms']
38
-
39
- Lines.log(args)
40
- end
41
-
42
- def identity(event)
43
- Lines.log(name: event.payload[:name], line: event.payload[:line])
44
- end
45
-
46
- def logger; Lines.logger; end
47
- end
48
- end
49
-
50
- # Subscribe lines to the AR events
51
- Lines::ActiveRecordSubscriber.attach_to :active_record
52
-
53
- # Remove the default ActiveRecord::LogSubscriber to avoid double outputs
54
- ActiveSupport::LogSubscriber.log_subscribers.each do |subscriber|
55
- if subscriber.is_a?(ActiveRecord::LogSubscriber)
56
- component = :active_record
57
- events = subscriber.public_methods(false).reject{ |method| method.to_s == 'call' }
58
- events.each do |event|
59
- ActiveSupport::Notifications.notifier.listeners_for("#{event}.#{component}").each do |listener|
60
- if listener.instance_variable_get('@delegate') == subscriber
61
- ActiveSupport::Notifications.unsubscribe listener
62
- end
63
- end
64
- end
65
- end
66
- end
67
-
68
- # Make sure it has a logger just in case
69
- ActiveRecord::Base.logger = Lines.logger
70
-
@@ -1,229 +0,0 @@
1
- module Lines
2
- module Error; end
3
-
4
- class Loader
5
- class ParseError < StandardError; include Error; end
6
-
7
- DOT = '.'
8
- EQUAL = '='
9
- SPACE = ' '
10
- OPEN_BRACKET = '['
11
- SHUT_BRACKET = ']'
12
- OPEN_BRACE = '{'
13
- SHUT_BRACE = '}'
14
- SINGLE_QUOTE = "'"
15
- DOUBLE_QUOTE = '"'
16
- BACKSLASH = '\\'
17
-
18
- ESCAPED_SINGLE_QUOTE = "\\'"
19
- ESCAPED_DOUBLE_QUOTE = '\"'
20
-
21
- LITERAL_MATCH = /[^=\s}\]]+/
22
- SINGLE_QUOTE_MATCH = /(?:\\.|[^'])*/
23
- DOUBLE_QUOTE_MATCH = /(?:\\.|[^"])*/
24
-
25
- NUM_MATCH = /-?(?:0|[1-9])\d*(?:\.\d+)?(?:[eE][+-]\d+)?/
26
- ISO8601_ZULU_CAPTURE = /^(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)Z$/
27
- NUM_CAPTURE = /^(#{NUM_MATCH})$/
28
- UNIT_CAPTURE = /^(#{NUM_MATCH}):(.+)/
29
-
30
- # Speeds parsing up a bit
31
- constants.each(&:freeze)
32
-
33
- EOF = nil
34
-
35
-
36
- def self.load(string)
37
- new.parse(string)
38
- end
39
-
40
- def parse(string)
41
- init(string.rstrip)
42
- inner_obj
43
- end
44
-
45
- protected
46
-
47
- def init(string)
48
- @string = string
49
- @pos = 0
50
- @c = @string[0]
51
- end
52
-
53
- def getc
54
- @pos += 1
55
- @c = @string[@pos]
56
- end
57
-
58
- def accept(char)
59
- if @c == char
60
- getc
61
- return true
62
- end
63
- false
64
- end
65
-
66
- def peek(num)
67
- @string[@pos+num]
68
- end
69
-
70
- def skip(num)
71
- @pos += num
72
- @c = @string[@pos]
73
- end
74
-
75
- def match(reg)
76
- @string.match(reg, @pos)
77
- end
78
-
79
- def expect(char)
80
- if !accept(char)
81
- fail "Expected '#{char}' but got '#{@c}'"
82
- end
83
- end
84
-
85
- def fail(msg)
86
- raise ParseError, "At #{@pos}, #{msg}"
87
- end
88
-
89
- def dbg(*x)
90
- #p [@pos, @c, @string[0..@pos]] + x
91
- end
92
-
93
- # Structures
94
-
95
-
96
- def inner_obj
97
- dbg :inner_obj
98
- # Shortcut for the '...' max_depth notation
99
- if @c == DOT && peek(1) == DOT && peek(2) == DOT
100
- expect DOT
101
- expect DOT
102
- expect DOT
103
- return {'...' => ''}
104
- end
105
-
106
- return {} if @c == EOF || @c == SHUT_BRACE
107
-
108
- # First pair
109
- k = key()
110
- expect EQUAL
111
- obj = {
112
- k => value()
113
- }
114
-
115
- while accept(SPACE)
116
- k = key()
117
- expect EQUAL
118
- obj[k] = value()
119
- end
120
-
121
- obj
122
- end
123
-
124
- def key
125
- dbg :key
126
-
127
- if @c == SINGLE_QUOTE
128
- single_quoted_string
129
- elsif @c == DOUBLE_QUOTE
130
- double_quoted_string
131
- else
132
- literal(false)
133
- end
134
- end
135
-
136
- def single_quoted_string
137
- dbg :single_quoted_string
138
-
139
- expect SINGLE_QUOTE
140
- md = match SINGLE_QUOTE_MATCH
141
- str = md[0].gsub ESCAPED_SINGLE_QUOTE, SINGLE_QUOTE
142
- skip md[0].size
143
-
144
- expect SINGLE_QUOTE
145
- str
146
- end
147
-
148
- def double_quoted_string
149
- dbg :double_quoted_string
150
-
151
- expect DOUBLE_QUOTE
152
- md = match DOUBLE_QUOTE_MATCH
153
- str = md[0].gsub ESCAPED_DOUBLE_QUOTE, DOUBLE_QUOTE
154
- skip md[0].size
155
-
156
- expect DOUBLE_QUOTE
157
- str
158
- end
159
-
160
- def literal(sub_parse)
161
- dbg :literal, sub_parse
162
-
163
- return "" unless ((md = match LITERAL_MATCH))
164
-
165
- literal = md[0]
166
- skip literal.size
167
-
168
- return literal unless sub_parse
169
-
170
- case literal
171
- when 'nil'
172
- nil
173
- when '#t'
174
- true
175
- when '#f'
176
- false
177
- when ISO8601_ZULU_CAPTURE
178
- Time.new($1.to_i, $2.to_i, $3.to_i, $4.to_i, $5.to_i, $6.to_i, '+00:00').utc
179
- when NUM_CAPTURE
180
- literal.index('.') ? Float(literal) : Integer(literal)
181
- when UNIT_CAPTURE
182
- num = $1.index('.') ? Float($1) : Integer($1)
183
- unit = $2
184
- [num, unit]
185
- else
186
- literal
187
- end
188
- end
189
-
190
- def value
191
- dbg :value
192
-
193
- case @c
194
- when OPEN_BRACKET
195
- list
196
- when OPEN_BRACE
197
- object
198
- when DOUBLE_QUOTE
199
- double_quoted_string
200
- when SINGLE_QUOTE
201
- single_quoted_string
202
- else
203
- literal(:sub_parse)
204
- end
205
- end
206
-
207
- def list
208
- dbg :list
209
-
210
- list = []
211
- expect(OPEN_BRACKET)
212
- list.push value
213
- while accept(SPACE)
214
- list.push value
215
- end
216
- expect(SHUT_BRACKET)
217
- list
218
- end
219
-
220
- def object
221
- dbg :object
222
-
223
- expect(OPEN_BRACE)
224
- obj = inner_obj
225
- expect(SHUT_BRACE)
226
- obj
227
- end
228
- end
229
- end