haproxy_log_parser 0.0.1

data/README.rdoc

@@ -0,0 +1,11 @@
+= haproxy_log_parser
+
+haproxy_log_parser is a gem that uses Treetop to parse HAProxy logs in
+HAProxy's HTTP log format.
+
+== Example
+
+  require 'haproxy_log_parser'
+  result = HAProxyLogParser.parse('Aug  9 20:30:46 localhost haproxy[2022]: 10.0.8.2:34028 [09/Aug/2011:20:30:46.429] proxy-out proxy-out/cache1 1/0/2/126/+128 301 +223 - - ---- 617/523/336/168/0 0/0 {www.sytadin.equipement.gouv.fr||http://trafic.1wt.eu/} {Apache|230|||http://www.sytadin.} "GET http://www.sytadin.equipement.gouv.fr/ HTTP/1.1"')
+  result.client_ip # => "10.0.8.2"
+  result.captured_response_headers # => ["Apache", "230", "", "", "http://www.sytadin."]

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/haproxy_log_parser.gemspec

@@ -0,0 +1,19 @@
+Gem::Specification.new do |s|
+  s.name = 'haproxy_log_parser'
+  s.version = IO.read('VERSION').chomp
+  s.authors = ['Toby Hsieh']
+  s.homepage = 'https://github.com/tobyhs/haproxy_log_parser'
+  s.summary = 'Parser for HAProxy logs in the HTTP log format'
+  s.description = s.summary
+
+  s.add_dependency 'treetop'
+
+  s.add_development_dependency 'rspec'
+
+  s.files = Dir.glob('lib/**/*') + [
+    'README.rdoc',
+    'VERSION',
+    'haproxy_log_parser.gemspec'
+  ]
+  s.test_files = Dir.glob('spec/**/*')
+end

data/lib/haproxy_log_parser.rb

@@ -0,0 +1,92 @@
+require 'treetop'
+
+require 'haproxy_log_parser/entry'
+
+Treetop.load(File.expand_path('haproxy_log_parser/line.treetop', File.dirname(__FILE__)))
+
+module HAProxyLogParser
+  VERSION = IO.read(File.join(File.dirname(__FILE__), '..', 'VERSION')).chomp.freeze
+
+  @parser = LineParser.new
+
+  class << self
+    # Returns an Entry object resulting from the given HAProxy HTTP-format log
+    # +line+, or +nil+ if the +line+ appears to be invalid.
+    #
+    # @param [String] line a line from an HAProxy log
+    # @return [Entry, nil]
+    def parse(line)
+      result = @parser.parse(line)
+      return nil unless result
+
+      entry = Entry.new
+      [
+        :client_ip, :frontend_name, :backend_name, :server_name,
+        :termination_state
+      ].each do |field|
+        entry.send("#{field}=", result.send(field).text_value)
+      end
+      [
+        :client_port, :tq, :tw, :tc, :tr, :tt, :status_code, :bytes_read,
+        :actconn, :feconn, :beconn, :srv_conn, :retries, :srv_queue,
+        :backend_queue
+      ].each do |field|
+        entry.send("#{field}=", result.send(field).text_value.to_i)
+      end
+
+      entry.accept_date = parse_accept_date(result.accept_date.text_value)
+      [:captured_request_cookie, :captured_response_cookie].each do |field|
+        cookie = decode_captured_cookie(result.send(field).text_value)
+        entry.send("#{field}=", cookie)
+      end
+      [:captured_request_headers, :captured_response_headers].each do |field|
+        headers = decode_captured_headers(result.send(field).text_value)
+        entry.send("#{field}=", headers)
+      end
+      entry.http_request = unescape(result.http_request.text_value)
+
+      entry
+    end
+
+    # Returns the given string un-escaped. See the "Logging > Non-printable
+    # characters" section in HAProxy documentation.
+    #
+    # @param [String] string
+    # @return [String]
+    def unescape(string)
+      string.gsub(/#[[:xdigit:]]{2}/) do |match|
+        match[1..-1].to_i(16).chr
+      end
+    end
+
+    # Converts the value of an accept_date field to a Time object.
+    #
+    # @param [String] string
+    # @return [Time]
+    def parse_accept_date(string)
+      parts = string.split(/[\/:.]/)
+      Time.local(*parts.values_at(2, 1, 0, 3..6))
+    end
+
+    # Converts a captured cookie string to a Hash.
+    #
+    # @param [String] string
+    # @return [Hash{String => String}]
+    def decode_captured_cookie(string)
+      if string == '-'
+        {}
+      else
+        key, value = string.split('=', 2)
+        {unescape(key) => unescape(value)}
+      end
+    end
+
+    # Converts a captured headers string to an Array.
+    #
+    # @param [String] string
+    # @return [Array<String>]
+    def decode_captured_headers(string)
+      string.split('|', -1).map! { |header| unescape(header) }
+    end
+  end
+end

data/lib/haproxy_log_parser/entry.rb

@@ -0,0 +1,84 @@
+module HAProxyLogParser
+  # An instance of this class represents a line/entry of an HAProxy log in the
+  # HTTP format. See the "Logging > Log formats > HTTP log format" section in
+  # HAProxy's configuration.txt for documentation of fields/attributes.
+  class Entry
+    # @return [String]
+    attr_accessor :client_ip
+
+    # @return [Integer]
+    attr_accessor :client_port
+
+    # @return [Time]
+    attr_accessor :accept_date
+
+    # @return [String]
+    attr_accessor :frontend_name
+
+    # @return [String]
+    attr_accessor :backend_name
+
+    # @return [String]
+    attr_accessor :server_name
+
+    # @return [Integer]
+    attr_accessor :tq
+
+    # @return [Integer]
+    attr_accessor :tw
+
+    # @return [Integer]
+    attr_accessor :tc
+
+    # @return [Integer]
+    attr_accessor :tr
+
+    # @return [Integer]
+    attr_accessor :tt
+
+    # @return [Integer]
+    attr_accessor :status_code
+
+    # @return [Integer]
+    attr_accessor :bytes_read
+
+    # @return [Hash{String => String}]
+    attr_accessor :captured_request_cookie
+
+    # @return [Hash{String => String}]
+    attr_accessor :captured_response_cookie
+
+    # @return [String]
+    attr_accessor :termination_state
+
+    # @return [Integer]
+    attr_accessor :actconn
+
+    # @return [Integer]
+    attr_accessor :feconn
+
+    # @return [Integer]
+    attr_accessor :beconn
+
+    # @return [Integer]
+    attr_accessor :srv_conn
+
+    # @return [Integer]
+    attr_accessor :retries
+
+    # @return [Integer]
+    attr_accessor :srv_queue
+
+    # @return [Integer]
+    attr_accessor :backend_queue
+
+    # @return [Array<String>]
+    attr_accessor :captured_request_headers
+
+    # @return [Array<String>]
+    attr_accessor :captured_response_headers
+
+    # @return [String]
+    attr_accessor :http_request
+  end
+end

data/lib/haproxy_log_parser/line.treetop

@@ -0,0 +1,58 @@
+module HAProxyLogParser
+  grammar Line
+    rule line
+      syslog_portion:([^\[]+ '[' integer ']: ')
+      client_ip:ip4_address ':' client_port:integer ' '
+      '[' accept_date '] '
+      frontend_name:proxy_name ' '
+      backend_name:proxy_name '/' server_name ' '
+      tq:integer '/' tw:integer '/' tc:integer '/' tr:integer '/' tt:integer ' '
+      status_code:integer ' '
+      bytes_read:integer ' '
+      captured_request_cookie:([^ ]+) ' '
+      captured_response_cookie:([^ ]+) ' '
+      termination_state ' '
+      actconn:integer '/' feconn:integer '/' beconn:integer '/'
+        srv_conn:integer '/' retries:integer ' '
+      srv_queue:integer '/' backend_queue:integer ' '
+      '{' captured_request_headers:captured_headers '} '
+      '{' captured_response_headers:captured_headers '} '
+      '"' http_request:[^"]+ '"'
+      "\n"?
+    end
+
+    rule integer
+      ('-' / '+')? [0-9]+
+    end
+
+    rule time
+      ([0-9] 2..2 ':') 2..2 ([0-9] 2..2)
+    end
+
+    rule ip4_address
+      ([0-9] 1..3 '.') 3..3 ([0-9] 1..3)
+    end
+
+    rule accept_date
+      [0-9] 2..2 '/' [A-Z] [a-z] 2..2 '/' [0-9]+ ':' time '.' ([0-9] 3..3)
+    end
+
+    rule proxy_name
+      [-_A-Za-z0-9.:]+
+    end
+
+    rule server_name
+      proxy_name / '<NOSRV>' / '<STATS>'
+    end
+
+    rule termination_state
+      [-CSPRIcs] [-RQCHDLT] [-NIDVEO] [-NIUPRD]
+    end
+
+    rule captured_headers
+      [^}]*
+    end
+  end
+end
+
+# vim:ai

data/spec/haproxy_log_parser_spec.rb

@@ -0,0 +1,72 @@
+require 'haproxy_log_parser'
+
+describe HAProxyLogParser do
+  # TODO Use something better instead of LINES[0], LINES[1], ...
+  LINES = IO.readlines(File.join(File.dirname(__FILE__), 'sample.log'))
+
+  describe '.parse' do
+    it 'parses LINE[0] correctly' do
+      entry = HAProxyLogParser.parse(LINES[0])
+      entry.client_ip.should == '10.0.8.2'
+      entry.client_port.should == 34028
+      entry.accept_date.should == Time.local(2011, 8, 9, 20, 30, 46, 429)
+      entry.frontend_name.should == 'proxy-out'
+      entry.backend_name.should == 'proxy-out'
+      entry.server_name.should == 'cache1'
+      entry.tq.should == 1
+      entry.tw.should == 0
+      entry.tc.should == 2
+      entry.tr.should == 126
+      entry.tt.should == 128
+      entry.status_code.should == 301
+      entry.bytes_read.should == 223
+      entry.captured_request_cookie.should == {}
+      entry.captured_response_cookie.should == {}
+      entry.termination_state.should == '----'
+      entry.actconn.should == 617
+      entry.feconn.should == 523
+      entry.beconn.should == 336
+      entry.srv_conn.should == 168
+      entry.retries.should == 0
+      entry.srv_queue.should == 0
+      entry.backend_queue.should == 0
+      entry.captured_request_headers.should == ['www.sytadin.equipement.gouv.fr', '', 'http://trafic.1wt.eu/']
+      entry.captured_response_headers.should == ['Apache', '230', '', '', 'http://www.sytadin.']
+      entry.http_request.should == 'GET http://www.sytadin.equipement.gouv.fr/ HTTP/1.1'
+    end
+
+    it 'parses LINES[1] correctly' do
+      entry = HAProxyLogParser.parse(LINES[1])
+      entry.client_ip.should == '192.168.1.215'
+      entry.client_port.should == 50679
+      entry.accept_date.should == Time.local(2012, 5, 21, 1, 35, 46, 146)
+      entry.frontend_name.should == 'webapp'
+      entry.backend_name.should == 'webapp_backend'
+      entry.server_name.should == 'web09'
+      entry.tq.should == 27
+      entry.tw.should == 0
+      entry.tc.should == 1
+      entry.tr.should == 0
+      entry.tt.should == 217
+      entry.status_code.should == 200
+      entry.bytes_read.should == 1367
+      entry.captured_request_cookie.should == {'session' => 'abc'}
+      entry.captured_response_cookie.should == {'session' => 'xyz'}
+      entry.termination_state.should == '----'
+      entry.actconn.should == 600
+      entry.feconn.should == 529
+      entry.beconn.should == 336
+      entry.srv_conn.should == 158
+      entry.retries.should == 0
+      entry.srv_queue.should == 0
+      entry.backend_queue.should == 0
+      entry.captured_request_headers.should == ['|| {5F41}', 'http://google.com/', '']
+      entry.captured_response_headers.should == ['1270925568', '', '']
+      entry.http_request.should == 'GET /images/image.gif HTTP/1.1'
+    end
+
+    it 'returns nil if the line is invalid' do
+      HAProxyLogParser.parse('asdf jkl;').should be_nil
+    end
+  end
+end

data/spec/sample.log

@@ -0,0 +1,2 @@
+Aug  9 20:30:46 localhost haproxy[2022]: 10.0.8.2:34028 [09/Aug/2011:20:30:46.429] proxy-out proxy-out/cache1 1/0/2/126/+128 301 +223 - - ---- 617/523/336/168/0 0/0 {www.sytadin.equipement.gouv.fr||http://trafic.1wt.eu/} {Apache|230|||http://www.sytadin.} "GET http://www.sytadin.equipement.gouv.fr/ HTTP/1.1"
+May 21 01:35:46 10.18.237.5 haproxy[26747]: 192.168.1.215:50679 [21/May/2012:01:35:46.146] webapp webapp_backend/web09 27/0/1/0/217 200 1367 session=abc session=xyz ---- 600/529/336/158/0 0/0 {#7C#7C #7B5F41#7D|http://google.com/|} {1270925568||} "GET /images/image.gif HTTP/1.1"

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification 
+name: haproxy_log_parser
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Toby Hsieh
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-12-19 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: treetop
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+description: Parser for HAProxy logs in the HTTP log format
+email: 
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/haproxy_log_parser/entry.rb
+- lib/haproxy_log_parser/line.treetop
+- lib/haproxy_log_parser.rb
+- README.rdoc
+- VERSION
+- haproxy_log_parser.gemspec
+- spec/haproxy_log_parser_spec.rb
+- spec/sample.log
+has_rdoc: true
+homepage: https://github.com/tobyhs/haproxy_log_parser
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Parser for HAProxy logs in the HTTP log format
+test_files: 
+- spec/haproxy_log_parser_spec.rb
+- spec/sample.log