heroku-log-parser 0.2 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 390fc3ebfd89128a6801b82c78651cc960a52c08
4
+ data.tar.gz: 16721fa3a81cb580cfc895e642593e9e2ec11f01
5
+ SHA512:
6
+ metadata.gz: 5adab4a76a2986527091ab2933e9126bbd8f1ef33a40322ba4a4f9c88a02ec3d14b90df48d407abd8b925c7c184524947065de74e7dc3c18d42f370bc38cc3ae
7
+ data.tar.gz: 7efd004c6933cfa519dd92f33b3b668580173bad38f0d5210d60b92316da8f6653c8783dfcb169a0e1132a5ac9204bed72f4ea64a4e7364be352477513db59d4
@@ -0,0 +1 @@
1
+ 2.1.7
data/Gemfile ADDED
@@ -0,0 +1,5 @@
1
+ source "https://rubygems.org"
2
+
3
+ gemspec
4
+
5
+ gem "rake"
@@ -0,0 +1,19 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ heroku-log-parser (0.2.2)
5
+
6
+ GEM
7
+ remote: https://rubygems.org/
8
+ specs:
9
+ rake (10.4.2)
10
+
11
+ PLATFORMS
12
+ ruby
13
+
14
+ DEPENDENCIES
15
+ heroku-log-parser!
16
+ rake
17
+
18
+ BUNDLED WITH
19
+ 1.10.6
data/README.md CHANGED
@@ -38,15 +38,11 @@ HerokuLogParser::SYSLOG_KEYS
38
38
 
39
39
  ## Contributions
40
40
 
41
+ * [Pablo Brasero](https://github.com/pablobm) for his work on keeping the parser up to date with Heroku's changing log format.
41
42
  * [Ryan Smith](https://github.com/ryandotsmith/) for his work on [l2met](https://github.com/ryandotsmith/l2met) which forms the foundation of heroku-log-parser.
42
43
 
43
- ## Todos
44
-
45
- * TESTS!!!!
46
- * 2nd order parsing. For instance, for parsing a structured message body into key=value pairs (including the structured_data message part)
47
-
48
44
  ## Issues
49
45
 
50
46
  Please submit all issues to the project's Github issues.
51
47
 
52
- -- [@rwdaigle](https://twitter.com/rwdaigle)
48
+ -- [@rwdaigle](https://twitter.com/rwdaigle)
@@ -0,0 +1,5 @@
1
+ require 'rake/testtask'
2
+
3
+ Rake::TestTask.new do |t|
4
+ t.pattern = "test/*_test.rb"
5
+ end
@@ -1,3 +1,5 @@
1
+ require 'time'
2
+
1
3
  class HerokuLogParser
2
4
 
3
5
  SYSLOG_KEYS = :priority, :syslog_version, :emitted_at, :hostname, :appname, :proc_id, :msg_id, :structured_data, :message
@@ -18,9 +20,9 @@ class HerokuLogParser
18
20
 
19
21
  # http://tools.ietf.org/html/rfc5424#page-8
20
22
  # frame <prority>version time hostname <appname-missing> procid msgid [no structured data = '-'] msg
21
- # 120 <40>1 2012-11-30T06:45:29+00:00 heroku web.3 d.73ea7440-270a-435a-a0ea-adf50b4e5f5a - State changed from starting to up
23
+ # 120 <40>1 2013-07-26T18:39:37.489572+00:00 host app web.11 - State changed from starting to up...
22
24
  def line_regex
23
- @line_regex ||= /\<(\d+)\>(1) (\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\+00:00) ([a-z0-9-]+) ([a-z0-9\-\_\.]+) ([a-z0-9\-\_\.]+) (\-) (.*)$/
25
+ @line_regex ||= /\<(\d+)\>(1) (\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d\d\d\d\d\d)?\+00:00) ([a-z0-9\-\_\.]+) ([a-z0-9\.-]+) ([a-z0-9\-\_\.]+) (\-) (.*)$/
24
26
  end
25
27
 
26
28
  # Heroku's http log drains (https://devcenter.heroku.com/articles/labs-https-drains)
@@ -37,13 +39,13 @@ class HerokuLogParser
37
39
  num_bytes = matching[1].to_i
38
40
  frame_offset = matching[0].length
39
41
  line_end = frame_offset + num_bytes
40
- msg = d[frame_offset..line_end]
42
+ msg = d[frame_offset..(line_end-1)]
41
43
  yield msg
42
44
  d = d[line_end..d.length]
43
45
  elsif matching = d.match(/\n/) # Newlines = explicit message delimiter
44
46
  d = matching.post_match
45
47
  else
46
- STDERR.puts("Unable to parse: #{d}")
48
+ STDERR.puts("Unable to parse: #{d}. Full line was: #{data_str.inspect}")
47
49
  return
48
50
  end
49
51
  end
@@ -56,11 +58,12 @@ class HerokuLogParser
56
58
  event[:syslog_version] = matching[2].to_i
57
59
  event[:emitted_at] = nil?(matching[3]) ? nil : Time.parse(matching[3]).utc
58
60
  event[:hostname] = interpret_nil(matching[4])
59
- event[:appname] = nil
60
- event[:proc_id] = interpret_nil(matching[5])
61
- event[:msg_id] = interpret_nil(matching[6])
61
+ event[:appname] = interpret_nil(matching[5])
62
+ event[:proc_id] = interpret_nil(matching[6])
63
+ event[:msg_id] = interpret_nil(nil)
62
64
  event[:structured_data] = interpret_nil(matching[7])
63
65
  event[:message] = interpret_nil(matching[8])
66
+ event[:original] = matching[0]
64
67
  event
65
68
  end
66
69
 
@@ -74,4 +77,4 @@ class HerokuLogParser
74
77
  val == "-"
75
78
  end
76
79
  end
77
- end
80
+ end
@@ -1,3 +1,3 @@
1
1
  class HerokuLogParser
2
- VERSION = "0.2" unless defined? HerokuLogParser::VERSION
3
- end
2
+ VERSION = "0.4.0" unless defined? HerokuLogParser::VERSION
3
+ end
@@ -0,0 +1,75 @@
1
+ require 'minitest/autorun'
2
+ require 'heroku-log-parser'
3
+
4
+ describe HerokuLogParser do
5
+
6
+ it "parses heroku log output" do
7
+ msg = "156 <40>1 2012-11-30T06:45:26+00:00 heroku web.3 d.73ea7440-270a-435a-a0ea-adf50b4e5f5a - Starting process with command `bundle exec rackup config.ru -p 24405`"
8
+ actual = HerokuLogParser.parse(msg)
9
+ expected = [{
10
+ priority: 40,
11
+ syslog_version: 1,
12
+ emitted_at: Time.parse("2012-11-30 06:45:26 UTC"),
13
+ hostname: "heroku",
14
+ appname: "web.3",
15
+ proc_id: "d.73ea7440-270a-435a-a0ea-adf50b4e5f5a",
16
+ msg_id: nil,
17
+ structured_data: nil,
18
+ message: "Starting process with command `bundle exec rackup config.ru -p 24405`",
19
+ original: "<40>1 2012-11-30T06:45:26+00:00 heroku web.3 d.73ea7440-270a-435a-a0ea-adf50b4e5f5a - Starting process with command `bundle exec rackup config.ru -p 24405`",
20
+ }]
21
+ assert_equal expected, actual
22
+ end
23
+
24
+ it "parses heroku postgres log output" do
25
+ msg = "530 <134>1 2016-02-13T21:20:25+00:00 host app heroku-postgres - source=DATABASE sample#current_transaction=15365 sample#db_size=4347350804bytes sample#tables=43 sample#active-connections=6 sample#waiting-connections=0 sample#index-cache-hit-rate=0.97116 sample#table-cache-hit-rate=0.73958 sample#load-avg-1m=0.05 sample#load-avg-5m=0.03 sample#load-avg-15m=0.035 sample#read-iops=0 sample#write-iops=112.73 sample#memory-total=15405636.0kB sample#memory-free=214004kB sample#memory-cached=14392920.0kB sample#memory-postgres=181644kB"
26
+ actual = HerokuLogParser.parse(msg)
27
+ expected = [{
28
+ priority: 134,
29
+ syslog_version: 1,
30
+ emitted_at: Time.parse("2016-02-13 21:20:25 UTC"),
31
+ hostname: "host",
32
+ appname: "app",
33
+ proc_id: "heroku-postgres",
34
+ msg_id: nil,
35
+ structured_data: nil,
36
+ message: "source=DATABASE sample#current_transaction=15365 sample#db_size=4347350804bytes sample#tables=43 sample#active-connections=6 sample#waiting-connections=0 sample#index-cache-hit-rate=0.97116 sample#table-cache-hit-rate=0.73958 sample#load-avg-1m=0.05 sample#load-avg-5m=0.03 sample#load-avg-15m=0.035 sample#read-iops=0 sample#write-iops=112.73 sample#memory-total=15405636.0kB sample#memory-free=214004kB sample#memory-cached=14392920.0kB sample#memory-postgres=181644kB",
37
+ original: "<134>1 2016-02-13T21:20:25+00:00 host app heroku-postgres - source=DATABASE sample#current_transaction=15365 sample#db_size=4347350804bytes sample#tables=43 sample#active-connections=6 sample#waiting-connections=0 sample#index-cache-hit-rate=0.97116 sample#table-cache-hit-rate=0.73958 sample#load-avg-1m=0.05 sample#load-avg-5m=0.03 sample#load-avg-15m=0.035 sample#read-iops=0 sample#write-iops=112.73 sample#memory-total=15405636.0kB sample#memory-free=214004kB sample#memory-cached=14392920.0kB sample#memory-postgres=181644kB",
38
+ }]
39
+ assert_equal expected, actual
40
+ end
41
+
42
+ it "parses multiple messages on the same line" do
43
+ # Real-world log
44
+ msg = %{200 <190>1 2014-07-17T12:59:25.223980+00:00 d.8f7a4b11-c323-c764-a74f-89bf4c1100ab app web.2 - - Started GET "/2013/02/26/full-text-search-in-your-browser" for 197.112.207.103 at 2014-07-17 12:59:25 +0000164 <191>1 2014-07-17T12:59:25.239432+00:00 d.8f7a4b11-c323-c764-a74f-89bf4c1100ab app web.1 - - Started GET "/blog.rss" for 54.219.13.100 at 2014-07-18 12:59:25 +0000}
45
+ actual = HerokuLogParser.parse(msg)
46
+ expected_first = {
47
+ priority: 190,
48
+ syslog_version: 1,
49
+ emitted_at: Time.parse('2014-07-17 12:59:25.223980 UTC'),
50
+ hostname: "d.8f7a4b11-c323-c764-a74f-89bf4c1100ab",
51
+ appname: "app",
52
+ proc_id: "web.2",
53
+ msg_id: nil,
54
+ structured_data: nil,
55
+ message: "- Started GET \"/2013/02/26/full-text-search-in-your-browser\" for 197.112.207.103 at 2014-07-17 12:59:25 +0000",
56
+ original: "<190>1 2014-07-17T12:59:25.223980+00:00 d.8f7a4b11-c323-c764-a74f-89bf4c1100ab app web.2 - - Started GET \"/2013/02/26/full-text-search-in-your-browser\" for 197.112.207.103 at 2014-07-17 12:59:25 +0000",
57
+ }
58
+ assert_equal expected_first, actual[0]
59
+
60
+ expected_second = {
61
+ priority: 191,
62
+ syslog_version: 1,
63
+ emitted_at: Time.parse('2014-07-17 12:59:25.239432 UTC'),
64
+ hostname: "d.8f7a4b11-c323-c764-a74f-89bf4c1100ab",
65
+ appname: "app",
66
+ proc_id: "web.1",
67
+ msg_id: nil,
68
+ structured_data: nil,
69
+ message: "- Started GET \"/blog.rss\" for 54.219.13.100 at 2014-07-18 12:59:25 +0000",
70
+ original: "<191>1 2014-07-17T12:59:25.239432+00:00 d.8f7a4b11-c323-c764-a74f-89bf4c1100ab app web.1 - - Started GET \"/blog.rss\" for 54.219.13.100 at 2014-07-18 12:59:25 +0000",
71
+ }
72
+ assert_equal expected_second, actual[1]
73
+ end
74
+
75
+ end
metadata CHANGED
@@ -1,15 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: heroku-log-parser
3
3
  version: !ruby/object:Gem::Version
4
- version: '0.2'
5
- prerelease:
4
+ version: 0.4.0
6
5
  platform: ruby
7
6
  authors:
8
7
  - Ryan Daigle
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2013-06-14 00:00:00.000000000 Z
11
+ date: 2017-08-28 00:00:00.000000000 Z
13
12
  dependencies: []
14
13
  description: Easily parse Heroku's syslog-based application log-stream
15
14
  email:
@@ -18,33 +17,38 @@ executables: []
18
17
  extensions: []
19
18
  extra_rdoc_files: []
20
19
  files:
21
- - .gitignore
20
+ - ".gitignore"
21
+ - ".ruby-version"
22
+ - Gemfile
23
+ - Gemfile.lock
22
24
  - README.md
25
+ - Rakefile
23
26
  - heroku-log-parser.gemspec
24
27
  - lib/heroku-log-parser.rb
25
28
  - lib/heroku-log-parser/version.rb
29
+ - test/heroku_log_parser_test.rb
26
30
  homepage: https://github.com/rwdaigle/heroku-log-parser
27
31
  licenses: []
32
+ metadata: {}
28
33
  post_install_message:
29
34
  rdoc_options: []
30
35
  require_paths:
31
36
  - lib
32
37
  required_ruby_version: !ruby/object:Gem::Requirement
33
- none: false
34
38
  requirements:
35
- - - ! '>='
39
+ - - ">="
36
40
  - !ruby/object:Gem::Version
37
41
  version: 1.8.7
38
42
  required_rubygems_version: !ruby/object:Gem::Requirement
39
- none: false
40
43
  requirements:
41
- - - ! '>='
44
+ - - ">="
42
45
  - !ruby/object:Gem::Version
43
46
  version: '0'
44
47
  requirements: []
45
48
  rubyforge_project: heroku-log-parser
46
- rubygems_version: 1.8.23
49
+ rubygems_version: 2.2.5
47
50
  signing_key:
48
- specification_version: 3
51
+ specification_version: 4
49
52
  summary: Syslog message parser
50
- test_files: []
53
+ test_files:
54
+ - test/heroku_log_parser_test.rb