heroku-log-parser 0.2 → 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 390fc3ebfd89128a6801b82c78651cc960a52c08
+  data.tar.gz: 16721fa3a81cb580cfc895e642593e9e2ec11f01
+SHA512:
+  metadata.gz: 5adab4a76a2986527091ab2933e9126bbd8f1ef33a40322ba4a4f9c88a02ec3d14b90df48d407abd8b925c7c184524947065de74e7dc3c18d42f370bc38cc3ae
+  data.tar.gz: 7efd004c6933cfa519dd92f33b3b668580173bad38f0d5210d60b92316da8f6653c8783dfcb169a0e1132a5ac9204bed72f4ea64a4e7364be352477513db59d4

data/.ruby-version

@@ -0,0 +1 @@
+2.1.7

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gemspec
+
+gem "rake"

data/Gemfile.lock

@@ -0,0 +1,19 @@
+PATH
+  remote: .
+  specs:
+    heroku-log-parser (0.2.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    rake (10.4.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  heroku-log-parser!
+  rake
+
+BUNDLED WITH
+   1.10.6

data/README.md

@@ -38,15 +38,11 @@ HerokuLogParser::SYSLOG_KEYS
 
 ## Contributions
 
+* [Pablo Brasero](https://github.com/pablobm) for his work on keeping the parser up to date with Heroku's changing log format.
 * [Ryan Smith](https://github.com/ryandotsmith/) for his work on [l2met](https://github.com/ryandotsmith/l2met) which forms the foundation of heroku-log-parser.
 
-## Todos
-
-* TESTS!!!!
-* 2nd order parsing. For instance, for parsing a structured message body into key=value pairs (including the structured_data message part)
-
 ## Issues
 
 Please submit all issues to the project's Github issues.
 
--- [@rwdaigle](https://twitter.com/rwdaigle)
+-- [@rwdaigle](https://twitter.com/rwdaigle)

data/Rakefile

@@ -0,0 +1,5 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.pattern = "test/*_test.rb"
+end

data/lib/heroku-log-parser.rb

@@ -1,3 +1,5 @@
+require 'time'
+
 class HerokuLogParser
 
   SYSLOG_KEYS = :priority, :syslog_version, :emitted_at, :hostname, :appname, :proc_id, :msg_id, :structured_data, :message
@@ -18,9 +20,9 @@ class HerokuLogParser
 
     # http://tools.ietf.org/html/rfc5424#page-8
     # frame <prority>version time hostname <appname-missing> procid msgid [no structured data = '-'] msg
-    # 120 <40>1 2012-11-30T06:45:29+00:00 heroku web.3 d.73ea7440-270a-435a-a0ea-adf50b4e5f5a - State changed from starting to up
+    # 120 <40>1 2013-07-26T18:39:37.489572+00:00 host app web.11 - State changed from starting to up...
     def line_regex
-      @line_regex ||= /\<(\d+)\>(1) (\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\+00:00) ([a-z0-9-]+) ([a-z0-9\-\_\.]+) ([a-z0-9\-\_\.]+) (\-) (.*)$/
+      @line_regex ||= /\<(\d+)\>(1) (\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d\d\d\d\d\d)?\+00:00) ([a-z0-9\-\_\.]+) ([a-z0-9\.-]+) ([a-z0-9\-\_\.]+) (\-) (.*)$/
     end
 
     # Heroku's http log drains (https://devcenter.heroku.com/articles/labs-https-drains)
@@ -37,13 +39,13 @@ class HerokuLogParser
           num_bytes = matching[1].to_i
           frame_offset = matching[0].length
           line_end = frame_offset + num_bytes
-          msg = d[frame_offset..line_end]
+          msg = d[frame_offset..(line_end-1)]
           yield msg
           d = d[line_end..d.length]
         elsif matching = d.match(/\n/) # Newlines = explicit message delimiter
           d = matching.post_match
         else
-          STDERR.puts("Unable to parse: #{d}")
+          STDERR.puts("Unable to parse: #{d}. Full line was: #{data_str.inspect}")
           return
         end
       end
@@ -56,11 +58,12 @@ class HerokuLogParser
       event[:syslog_version] = matching[2].to_i
       event[:emitted_at] = nil?(matching[3]) ? nil : Time.parse(matching[3]).utc
       event[:hostname] = interpret_nil(matching[4])
-      event[:appname] = nil
-      event[:proc_id] = interpret_nil(matching[5])
-      event[:msg_id] = interpret_nil(matching[6])
+      event[:appname] = interpret_nil(matching[5])
+      event[:proc_id] = interpret_nil(matching[6])
+      event[:msg_id] = interpret_nil(nil)
       event[:structured_data] = interpret_nil(matching[7])
       event[:message] = interpret_nil(matching[8])
+      event[:original] = matching[0]
       event
     end
 
@@ -74,4 +77,4 @@ class HerokuLogParser
       val == "-"
     end
   end
-end
+end

data/lib/heroku-log-parser/version.rb

@@ -1,3 +1,3 @@
 class HerokuLogParser
-  VERSION = "0.2" unless defined? HerokuLogParser::VERSION
-end
+  VERSION = "0.4.0" unless defined? HerokuLogParser::VERSION
+end

data/test/heroku_log_parser_test.rb

@@ -0,0 +1,75 @@
+require 'minitest/autorun'
+require 'heroku-log-parser'
+
+describe HerokuLogParser do
+
+  it "parses heroku log output" do
+    msg = "156 <40>1 2012-11-30T06:45:26+00:00 heroku web.3 d.73ea7440-270a-435a-a0ea-adf50b4e5f5a - Starting process with command `bundle exec rackup config.ru -p 24405`"
+    actual = HerokuLogParser.parse(msg)
+    expected = [{
+      priority: 40,
+      syslog_version: 1,
+      emitted_at: Time.parse("2012-11-30 06:45:26 UTC"),
+      hostname: "heroku",
+      appname: "web.3",
+      proc_id: "d.73ea7440-270a-435a-a0ea-adf50b4e5f5a",
+      msg_id: nil,
+      structured_data: nil,
+      message: "Starting process with command `bundle exec rackup config.ru -p 24405`",
+      original: "<40>1 2012-11-30T06:45:26+00:00 heroku web.3 d.73ea7440-270a-435a-a0ea-adf50b4e5f5a - Starting process with command `bundle exec rackup config.ru -p 24405`",
+    }]
+    assert_equal expected, actual
+  end
+
+  it "parses heroku postgres log output" do
+    msg = "530 <134>1 2016-02-13T21:20:25+00:00 host app heroku-postgres - source=DATABASE sample#current_transaction=15365 sample#db_size=4347350804bytes sample#tables=43 sample#active-connections=6 sample#waiting-connections=0 sample#index-cache-hit-rate=0.97116 sample#table-cache-hit-rate=0.73958 sample#load-avg-1m=0.05 sample#load-avg-5m=0.03 sample#load-avg-15m=0.035 sample#read-iops=0 sample#write-iops=112.73 sample#memory-total=15405636.0kB sample#memory-free=214004kB sample#memory-cached=14392920.0kB sample#memory-postgres=181644kB"
+    actual = HerokuLogParser.parse(msg)
+    expected = [{
+      priority: 134,
+      syslog_version: 1,
+      emitted_at: Time.parse("2016-02-13 21:20:25 UTC"),
+      hostname: "host",
+      appname: "app",
+      proc_id: "heroku-postgres",
+      msg_id: nil,
+      structured_data: nil,
+      message: "source=DATABASE sample#current_transaction=15365 sample#db_size=4347350804bytes sample#tables=43 sample#active-connections=6 sample#waiting-connections=0 sample#index-cache-hit-rate=0.97116 sample#table-cache-hit-rate=0.73958 sample#load-avg-1m=0.05 sample#load-avg-5m=0.03 sample#load-avg-15m=0.035 sample#read-iops=0 sample#write-iops=112.73 sample#memory-total=15405636.0kB sample#memory-free=214004kB sample#memory-cached=14392920.0kB sample#memory-postgres=181644kB",
+      original: "<134>1 2016-02-13T21:20:25+00:00 host app heroku-postgres - source=DATABASE sample#current_transaction=15365 sample#db_size=4347350804bytes sample#tables=43 sample#active-connections=6 sample#waiting-connections=0 sample#index-cache-hit-rate=0.97116 sample#table-cache-hit-rate=0.73958 sample#load-avg-1m=0.05 sample#load-avg-5m=0.03 sample#load-avg-15m=0.035 sample#read-iops=0 sample#write-iops=112.73 sample#memory-total=15405636.0kB sample#memory-free=214004kB sample#memory-cached=14392920.0kB sample#memory-postgres=181644kB",
+    }]
+    assert_equal expected, actual
+  end
+
+  it "parses multiple messages on the same line" do
+    # Real-world log
+    msg = %{200 <190>1 2014-07-17T12:59:25.223980+00:00 d.8f7a4b11-c323-c764-a74f-89bf4c1100ab app web.2 - - Started GET "/2013/02/26/full-text-search-in-your-browser" for 197.112.207.103 at 2014-07-17 12:59:25 +0000164 <191>1 2014-07-17T12:59:25.239432+00:00 d.8f7a4b11-c323-c764-a74f-89bf4c1100ab app web.1 - - Started GET "/blog.rss" for 54.219.13.100 at 2014-07-18 12:59:25 +0000}
+    actual = HerokuLogParser.parse(msg)
+    expected_first = {
+      priority: 190,
+      syslog_version: 1,
+      emitted_at: Time.parse('2014-07-17 12:59:25.223980 UTC'),
+      hostname: "d.8f7a4b11-c323-c764-a74f-89bf4c1100ab",
+      appname: "app",
+      proc_id: "web.2",
+      msg_id: nil,
+      structured_data: nil,
+      message: "- Started GET \"/2013/02/26/full-text-search-in-your-browser\" for 197.112.207.103 at 2014-07-17 12:59:25 +0000",
+      original: "<190>1 2014-07-17T12:59:25.223980+00:00 d.8f7a4b11-c323-c764-a74f-89bf4c1100ab app web.2 - - Started GET \"/2013/02/26/full-text-search-in-your-browser\" for 197.112.207.103 at 2014-07-17 12:59:25 +0000",
+    }
+    assert_equal expected_first, actual[0]
+
+    expected_second = {
+      priority: 191,
+      syslog_version: 1,
+      emitted_at: Time.parse('2014-07-17 12:59:25.239432 UTC'),
+      hostname: "d.8f7a4b11-c323-c764-a74f-89bf4c1100ab",
+      appname: "app",
+      proc_id: "web.1",
+      msg_id: nil,
+      structured_data: nil,
+      message: "- Started GET \"/blog.rss\" for 54.219.13.100 at 2014-07-18 12:59:25 +0000",
+      original: "<191>1 2014-07-17T12:59:25.239432+00:00 d.8f7a4b11-c323-c764-a74f-89bf4c1100ab app web.1 - - Started GET \"/blog.rss\" for 54.219.13.100 at 2014-07-18 12:59:25 +0000",
+    }
+    assert_equal expected_second, actual[1]
+  end
+
+end

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: heroku-log-parser
 version: !ruby/object:Gem::Version
-  version: '0.2'
-  prerelease: 
+  version: 0.4.0
 platform: ruby
 authors:
 - Ryan Daigle
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-14 00:00:00.000000000 Z
+date: 2017-08-28 00:00:00.000000000 Z
 dependencies: []
 description: Easily parse Heroku's syslog-based application log-stream
 email:
@@ -18,33 +17,38 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
 - README.md
+- Rakefile
 - heroku-log-parser.gemspec
 - lib/heroku-log-parser.rb
 - lib/heroku-log-parser/version.rb
+- test/heroku_log_parser_test.rb
 homepage: https://github.com/rwdaigle/heroku-log-parser
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.8.7
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: heroku-log-parser
-rubygems_version: 1.8.23
+rubygems_version: 2.2.5
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Syslog message parser
-test_files: []
+test_files:
+- test/heroku_log_parser_test.rb