heroku-log-parser 0.2

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+
+.rvmrc

data/README.md

@@ -0,0 +1,52 @@
+heroku-log-parser
+=======
+
+A [syslog (rfc5424)](http://tools.ietf.org/html/rfc5424#section-6) parser written in Ruby and specifically
+targeting Heroku's [http log drain](https://devcenter.heroku.com/articles/labs-https-drains).
+
+## Install
+
+Declare `heroku-log-parser` in your `Gemfile`.
+
+```ruby
+gem 'heroku-log-parser', :git => 'git@github.com:rwdaigle/heroku-log-parser.git'
+```
+
+Run bundler.
+
+```term
+$ bundle install
+```
+
+## Usage
+
+```ruby
+msg_str = "156 <40>1 2012-11-30T06:45:26+00:00 heroku web.3 d.73ea7440-270a-435a-a0ea-adf50b4e5f5a - Starting process with command `bundle exec rackup config.ru -p 24405`"
+
+HerokuLogParser.parse(msg_str)
+#=> [{:priority=>40, :syslog_version=>1, :emitted_at=>2012-11-30 06:45:26 UTC, :hostname=>"heroku", :appname=>nil, :proc_id=>"web.3", :msg_id=>"d.73ea7440-270a-435a-a0ea-adf50b4e5f5a", :structured_data=>nil, :message=>"Starting process with command `bundle exec rackup config.ru -p 24405`"}]
+```
+
+`HerokuLogParser` is a stateless, regex-based parser that accepts a string of data holding one or more syslog messages
+and returns an array of syslog message properties for each message. For those unwilling to read the spec, the
+list of syslog tokens is as follows (and is stored in the `HerokuLogParser::SYSLOG_KEYS` array):
+
+```ruby
+HerokuLogParser::SYSLOG_KEYS
+#=> [:priority, :syslog_version, :emitted_at, :hostname, :appname, :proc_id, :msg_id, :structured_data, :message]
+```
+
+## Contributions
+
+* [Ryan Smith](https://github.com/ryandotsmith/) for his work on [l2met](https://github.com/ryandotsmith/l2met) which forms the foundation of heroku-log-parser.
+
+## Todos
+
+* TESTS!!!!
+* 2nd order parsing. For instance, for parsing a structured message body into key=value pairs (including the structured_data message part)
+
+## Issues
+
+Please submit all issues to the project's Github issues.
+
+-- [@rwdaigle](https://twitter.com/rwdaigle)

data/heroku-log-parser.gemspec

@@ -0,0 +1,52 @@
+$LOAD_PATH.push File.expand_path("../lib", __FILE__)
+require 'heroku-log-parser/version'
+
+Gem::Specification.new do |s|
+  s.name              = "heroku-log-parser"
+  s.version           = HerokuLogParser::VERSION
+  s.platform          = Gem::Platform::RUBY
+  s.author            = "Ryan Daigle"
+  s.email             = ["ryan.daigle@gmail.com"]
+  s.homepage          = "https://github.com/rwdaigle/heroku-log-parser"
+  s.summary           = "Syslog message parser"
+  s.description       = "Easily parse Heroku's syslog-based application log-stream"
+
+  s.rubyforge_project = "heroku-log-parser"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  if File.exists?('UPGRADING')
+    s.post_install_message = File.read("UPGRADING")
+  end
+
+  s.required_ruby_version = ">= 1.8.7"
+
+  # s.add_dependency('activerecord', '>= 3.0.0')
+  # s.add_dependency('activemodel', '>= 3.0.0')
+  # s.add_dependency('activesupport', '>= 3.0.0')
+  # s.add_dependency('cocaine', '~> 0.4.0')
+  # s.add_dependency('mime-types')
+
+  # s.add_development_dependency('shoulda')
+  # s.add_development_dependency('appraisal')
+  # s.add_development_dependency('mocha')
+  # s.add_development_dependency('aws-sdk', '>= 1.2.0')
+  # s.add_development_dependency('bourne')
+  # s.add_development_dependency('sqlite3', '~> 1.3.4')
+  # s.add_development_dependency('cucumber', '~> 1.2.1')
+  # s.add_development_dependency('aruba')
+  # s.add_development_dependency('nokogiri')
+  # s.add_development_dependency('capybara')
+  # s.add_development_dependency('bundler')
+  # s.add_development_dependency('cocaine', '~> 0.2')
+  # s.add_development_dependency('fog', '>= 1.4.0', "< 1.7.0")
+  # s.add_development_dependency('pry')
+  # s.add_development_dependency('launchy')
+  # s.add_development_dependency('rake')
+  # s.add_development_dependency('fakeweb')
+  # s.add_development_dependency('railties')
+  # s.add_development_dependency('actionmailer')
+end

data/lib/heroku-log-parser.rb

@@ -0,0 +1,77 @@
+class HerokuLogParser
+
+  SYSLOG_KEYS = :priority, :syslog_version, :emitted_at, :hostname, :appname, :proc_id, :msg_id, :structured_data, :message
+
+  class << self
+
+    def parse(data_str)
+      events = []
+      lines(data_str) do |line|
+        if(matching = line.match(line_regex))
+          events << event_data(matching)
+        end
+      end
+      events
+    end
+
+    protected
+
+    # http://tools.ietf.org/html/rfc5424#page-8
+    # frame <prority>version time hostname <appname-missing> procid msgid [no structured data = '-'] msg
+    # 120 <40>1 2012-11-30T06:45:29+00:00 heroku web.3 d.73ea7440-270a-435a-a0ea-adf50b4e5f5a - State changed from starting to up
+    def line_regex
+      @line_regex ||= /\<(\d+)\>(1) (\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\+00:00) ([a-z0-9-]+) ([a-z0-9\-\_\.]+) ([a-z0-9\-\_\.]+) (\-) (.*)$/
+    end
+
+    # Heroku's http log drains (https://devcenter.heroku.com/articles/labs-https-drains)
+    # utilize octet counting framing (http://tools.ietf.org/html/draft-gerhards-syslog-plain-tcp-12#section-3.4.1)
+    # for transmission of syslog messages over TCP. Properly parse and delimit
+    # individual syslog messages, many of which may be contained in a single packet.
+    #
+    # I am still uncertain if this is the place for transport layer protocol handling. I suspect not.
+    #
+    def lines(data_str, &block)
+      d = data_str
+      while d && d.length > 0
+        if matching = d.match(/^(\d+) /) # if have a counting frame, use it
+          num_bytes = matching[1].to_i
+          frame_offset = matching[0].length
+          line_end = frame_offset + num_bytes
+          msg = d[frame_offset..line_end]
+          yield msg
+          d = d[line_end..d.length]
+        elsif matching = d.match(/\n/) # Newlines = explicit message delimiter
+          d = matching.post_match
+        else
+          STDERR.puts("Unable to parse: #{d}")
+          return
+        end
+      end
+    end
+
+    # Heroku is missing the appname token, otherwise can treat as standard syslog format
+    def event_data(matching)
+      event = {}
+      event[:priority] = matching[1].to_i
+      event[:syslog_version] = matching[2].to_i
+      event[:emitted_at] = nil?(matching[3]) ? nil : Time.parse(matching[3]).utc
+      event[:hostname] = interpret_nil(matching[4])
+      event[:appname] = nil
+      event[:proc_id] = interpret_nil(matching[5])
+      event[:msg_id] = interpret_nil(matching[6])
+      event[:structured_data] = interpret_nil(matching[7])
+      event[:message] = interpret_nil(matching[8])
+      event
+    end
+
+    private
+
+    def interpret_nil(val)
+      nil?(val) ? nil : val
+    end
+
+    def nil?(val)
+      val == "-"
+    end
+  end
+end

data/lib/heroku-log-parser/version.rb

@@ -0,0 +1,3 @@
+class HerokuLogParser
+  VERSION = "0.2" unless defined? HerokuLogParser::VERSION
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: heroku-log-parser
+version: !ruby/object:Gem::Version
+  version: '0.2'
+  prerelease: 
+platform: ruby
+authors:
+- Ryan Daigle
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-14 00:00:00.000000000 Z
+dependencies: []
+description: Easily parse Heroku's syslog-based application log-stream
+email:
+- ryan.daigle@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- README.md
+- heroku-log-parser.gemspec
+- lib/heroku-log-parser.rb
+- lib/heroku-log-parser/version.rb
+homepage: https://github.com/rwdaigle/heroku-log-parser
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.8.7
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: heroku-log-parser
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Syslog message parser
+test_files: []