haproxy2rpm 0.0.2 → 0.0.3

data/README.md

@@ -1,6 +1,8 @@
 # Push haproxy logs to newrelic rpm
 This is useful for languages where newrelic does not provide an agent,
-such as erlang or node.js based applications
+such as erlang or node.js based applications.
+
+It runs in syslog mode or tail a log file.
 
 ## Installation
 
@@ -9,26 +11,22 @@ gem install haproxy2rpm or clone it
 * copy your newrelic.yml file to $HOME/.newrelic/newrelic.yml
 * or set $NRCONFIG to point to your newrelic.yml file
 
-## Running it
-haproxy2rpm /path/to/logfile or ./bin/haproxy2rpm /path/to/log/file
-
-## Analyzing it
-
-At the moment, it only works with custom views
+Tell haproxy to log to syslog. e.g:
 
-<verbatim>
-  <h3>Charts</h3>
-  <table width='100%'>
-    <tr>
-    <td>line_chart {% line_chart value:'average_value' title:'Test' metric:'Custom/HAProxy/response_times' %}</td>
-  </tr>
+    # /etc/haproxy/haproxy.cfg
+    global
+      log 127.0.0.1:3333   daemon
 
-  </table>
-</verbatim/>
-
-
-## Roadmap
-
-* daemonize option
-* syslog (udp maybe tcp) server with https://github.com/loggly/logporter and then point HaProxy to that port. Why touch the disk if we don't have to?
-* Figure out how to report rpms and response times so that they show up inside the newrelic application view and not only as a custom metric
+## Running it
+    haproxy2rpm /path/to/logfile 
+    haproxy2rpm --syslog
+    haproxy2rpm --syslog --daemonize
+
+## Supported RPM features
+
+* Response times in application server
+* Error rate
+* Queue time
+* Apdex
+* Histogram
+* Web transactions

data/Rakefile

@@ -11,4 +11,14 @@ namespace :test do
 
     t.verbose = true
   end
+
+  desc 'send udp log lines for testing purposes'
+  task :send_udp_log_lines do
+    port = ENV['port'] || 3333
+    status_code = ENV['status_code'] || 200
+    uri = ENV['uri'] || '/'
+    puts 'make sure the udp server is running'
+
+    `yes '<13>May 19 18:30:17 localhost haproxy[674]: 127.0.0.1:33319 [15/Oct/2003:08:31:57] relais-http Srv1 6559/100/7/147/6723 #{status_code} 243 - - ---- 1/3/5 0/0 "PUT #{uri} HTTP/1.0"' | nc -u -i 1 localhost #{port}`
+  end
 end

data/Research.md

@@ -0,0 +1,57 @@
+# Agent#record_transaction
+# lib/new_relic/agent.rb
+# NewRelic::Agent.record_transaction( params['value'].to_f, params )
+# lib/new_relic/rack.rb
+# test/rpm_agent_test.rb
+# NewRelic::Agent.record_transaction 0.5, 'uri' => "/users/create?foo=bar"
+
+# NewRelic::Agent.record_transaction rand(100) / 100.0, {'metric' => 'Controller/josef'}
+# stat = NewRelic::Agent.agent.stats_engine.get_stats_no_scope 'WebFrontend/QueueTime'
+# stat.record_data_point(0.5)
+
+module NewRelic
+  module Metrics
+    CONTROLLER = "Controller"
+    DISPATCHER = "HttpDispatcher"
+    ACTIVE_RECORD = "ActiveRecord"
+    USER_TIME = "CPU/User Time"
+    MEMORY = "Memory/Physical"
+  end
+end
+
+def record_transaction(duration_seconds, options={})
+  is_error = options['is_error'] || options['error_message'] ||
+options['exception']
+  metric = options['metric']
+  metric ||= options['uri'] # normalize this with url rules
+  raise "metric or uri arguments required" unless metric
+  metric_info =
+NewRelic::MetricParser::MetricParser.for_metric_named(metric)
+
+  if metric_info.is_web_transaction?
+    NewRelic::Agent::Instrumentation::MetricFrame.record_apdex(metric_info,
+duration_seconds, duration_seconds, is_error)
+    histogram.process(duration_seconds)
+  end
+  metrics = metric_info.summary_metrics
+
+  metrics << metric
+  metrics.each do |name|
+    stats = stats_engine.get_stats_no_scope(name)
+    stats.record_data_point(duration_seconds)
+  end
+
+  if is_error
+    if options['exception']
+      e = options['exception']
+    elsif options['error_message']
+      e = Exception.new options['error_message']
+    else
+      e = Exception.new 'Unknown Error'
+    end
+    error_collector.notice_error e, :uri => options['uri'],
+:metric => metric
+  end
+  # busy time ?
+end
+

data/bin/haproxy2rpm

@@ -10,23 +10,49 @@ NEW_RELIC_CONFIG_PATH = File.join(ENV['HOME'], '.newrelic', 'newrelic.yml')
 help = <<HELP
 haproxy2rpm
 
-Basic Command Line Usage:
-  haproxy2rpm <haproxy log file to parse>
-
-  Coniguration for newrelic is read from #{NEW_RELIC_CONFIG_PATH}
+Coniguration for newrelic is read from #{NEW_RELIC_CONFIG_PATH}
 HELP
 
-options = { :daemonize => false, :version => false, :syslog => false }
+options = {
+  :daemonize => false,
+  :version => false,
+  :syslog => false,
+  :port => 3333,
+  :listen => '0.0.0.0'
+}
 
 opts = OptionParser.new do |opts|
-  opts.banner = help
+#  opts.banner = help
   opts.on("-D", "--daemonize", "Daemonize") do
     options[:daemonize] = true
   end
 
   opts.on("-s", "--syslog", "Run syslog server") do
     options[:syslog] = true
-  end  
+  end
+
+  opts.on("-lIP", "--listen IP", "Set IP to listen on") do |ip|
+    options[:listen] = ip
+  end
+
+  opts.on("-pPORT", "--port PORT", "Set port to listen on") do |port|
+    options[:port] = port
+  end
+
+  opts.on("-nAPP_NAME", "--app_name APP_NAME", "Set application name") do |app_name|
+    options[:app_name] = app_name
+  end
+
+  opts.on("-eENVIRONMENT", "--environment ENVIRONMENT", "Set Newrelic agent env") do |env|
+    options[:env] = env
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts.banner
+    puts opts
+    exit
+  end
+
   opts.on("-v", "--version", "Print the version number and exit") do
     options[:version] = true
   end
@@ -47,6 +73,7 @@ unless options[:syslog] || File.exists?(log_file)
   puts 'please proivde a valid path to a haproxy log file'
   puts ''
   puts help
+  puts opts
   exit(1)
 end
 

data/lib/haproxy2rpm.rb

@@ -11,7 +11,7 @@ require "haproxy2rpm/rpm"
 
 module Haproxy2Rpm
   def self.run(log_file, options)
-    @rpm = Rpm.new
+    @rpm = Rpm.new(options)
     if(options[:daemonize])
       puts 'daemonizing'
       run_daemonized(log_file, options)
@@ -28,14 +28,13 @@ module Haproxy2Rpm
     puts 'stopping new relic agent'
     NewRelic::Agent.shutdown
   end
-  
+
   def self.run_syslog_server(options)
-    NewRelic::Agent.manual_start
     EventMachine::run do
-      EventMachine.start_server("0.0.0.0", 3333, SyslogHandler)
-    end      
+      EventMachine::open_datagram_socket(options[:listen], options[:port], SyslogHandler)
+    end
   end
-  
+
   def self.default_run(log_file,options)
     EventMachine.run do
       EventMachine::file_tail(log_file) do |filetail, line|
@@ -69,4 +68,4 @@ module Haproxy2Rpm
       
       exit
   end
-end
+end

data/lib/haproxy2rpm/line_parser.rb

@@ -36,7 +36,19 @@ module Haproxy2Rpm
     end
 
     def uri
-      @uri ||= @parts[14]
+      @uri ||= URI.parse(@parts[14])
+    end
+
+    def http_path
+      uri.path
+    end
+
+    def http_query
+      uri.query
+    end
+
+    def is_error?
+      status_code >= 500
     end
 
     private

data/lib/haproxy2rpm/rpm.rb

@@ -1,13 +1,23 @@
 module Haproxy2Rpm
   class Rpm
-    def initialize()
-      NewRelic::Agent.manual_start
+    def initialize(options = {})
+      agent_options = {}
+      agent_options[:app_name] = options[:app_name] if options[:app_name]
+      agent_options[:env] = options[:env] if options[:env]
+      NewRelic::Agent.manual_start agent_options
       @stats_engine = NewRelic::Agent.agent.stats_engine
+      @qt_stat = @stats_engine.get_stats_no_scope('WebFrontend/QueueTime')
     end
-    
+
     def send(line)
       request = LineParser.new(line)
-      @stats_engine.get_stats('Custom/HAProxy/response_times',false).record_data_point(request.tr)
+      NewRelic::Agent.record_transaction(
+        request.tr / 1000.0,
+        'metric' => "Controller#{request.path}",
+        'is_error' => request.is_error?,
+        'error_message' => "#{request.uri} : Status code #{request.status_code}"
+      )
+      @qt_stat.record_data_point(request.tw / 1000.0)
     end
   end
-end
+end

data/lib/haproxy2rpm/syslog.rb

@@ -29,22 +29,12 @@ module Haproxy2Rpm
       end
 
       @syslog3164_re = Regexp.new(re)
-    end # def initialize
+    end
 
     def receive_data(data)
-      # In jruby+netty, we probaby should use the DelimiterBasedFrameDecoder
-      # But for the sake of simplicity, we'll use EM's BufferedTokenizer for
-      # all implementations.
-      @buffer.extract(data).each do |line|
-        receive_line(line.chomp)
-      end
-    end # def receive_event
-
-    def receive_line(line)
-      @count += 1
-      # Just try matching, don't need to do anything with it for this benchmark.
-      m = @syslog3164_re.match(line)
+      puts data
+      m = @syslog3164_re.match(data)
       @rpm.send(m[:message])
     end
   end
-end
+end

data/lib/haproxy2rpm/version.rb

@@ -1,3 +1,3 @@
 module Haproxy2Rpm
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/test/haproxy2pm_test.rb

@@ -2,11 +2,28 @@ $LOAD_PATH.unshift( File.dirname(__FILE__) )
 
 require 'test_helper'
 
+def log_entry(options = {})
+  defaults = {
+                :tq => 6559,
+                :tw => 100,
+                :tc => 7,
+                :tr => 147,
+                :tt => 6723,
+                :status_code => 200,
+                :http_path => '/',
+                :http_method => 'GET',
+                :http_query => 'example_param=test',
+            }
+  defaults.merge!(options)
+  log_line = <<LOG_LINE
+haproxy[674]: 127.0.0.1:33319 [15/Oct/2003:08:31:57] relais-http Srv1 #{defaults[:tq]}/#{defaults[:tw]}/#{defaults[:tc]}/#{defaults[:tr]}/#{defaults[:tt]} #{defaults[:status_code]} 243 - - ---- 1/3/5 0/0 "#{defaults[:http_method]} #{defaults[:http_path]}#{defaults[:http_query] ? "?#{defaults[:http_query]}" : ''} HTTP/1.0"
+LOG_LINE
+end
+
 class Haproxy2RpmTest < Test::Unit::TestCase
   context 'parsing of haproxy files' do
     setup do
-      @line = File.open(File.join(File.dirname(__FILE__), 'fixtures', 'haproxy.log')){|f| f.readlines}.first
-      @result = Haproxy2Rpm::LineParser.new(@line)
+      @result = Haproxy2Rpm::LineParser.new(log_entry)
     end
 
     should 'parse the Tq (total time in ms spent waiting for client)' do
@@ -15,7 +32,7 @@ class Haproxy2RpmTest < Test::Unit::TestCase
 
     # this is the time waiting in the global queue
     should 'parse the Tw (total time in ms spent waiting in queue)' do
-      assert_equal 0, @result.tw
+      assert_equal 100, @result.tw
     end
 
     should 'parse the Tc (total time in ms spent waiting for the connection to the final server' do
@@ -36,11 +53,34 @@ class Haproxy2RpmTest < Test::Unit::TestCase
     end
 
     should 'parse the http method' do
-      assert_equal 'HEAD', @result.http_method
+      assert_equal 'GET', @result.http_method
     end
 
     should 'parse the uri' do
-      assert_equal '/', @result.uri
+      assert_equal '/', @result.http_path
+    end
+
+    should 'parse the GET params' do
+      assert_equal 'example_param=test', @result.http_query
+    end
+  end
+
+  context 'is_error' do
+   should 'return false for redirects' do
+      assert !Haproxy2Rpm::LineParser.new(log_entry(:status_code => 302)).is_error?
     end
+
+    should 'return false for 200 OK' do
+      assert !Haproxy2Rpm::LineParser.new(log_entry(:status_code => 200)).is_error?
+    end
+
+    should 'return false for 404' do
+      assert !Haproxy2Rpm::LineParser.new(log_entry(:status_code => 404)).is_error?
+    end
+
+    should 'return for 500 and above' do
+      assert Haproxy2Rpm::LineParser.new(log_entry(:status_code => 500)).is_error?
+    end
+
   end
 end

metadata

@@ -1,76 +1,74 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: haproxy2rpm
-version: !ruby/object:Gem::Version 
+version: !ruby/object:Gem::Version
+  version: 0.0.3
   prerelease: 
-  version: 0.0.2
 platform: ruby
-authors: 
+authors:
 - Patrick Huesler
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-07-09 00:00:00 +02:00
+date: 2011-07-11 00:00:00.000000000 +02:00
 default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+dependencies:
+- !ruby/object:Gem::Dependency
   name: newrelic_rpm
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &2153806900 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: eventmachine-tail
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *2153806900
+- !ruby/object:Gem::Dependency
+  name: eventmachine-tail
+  requirement: &2153806480 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: rake
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *2153806480
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &2153806060 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: shoulda-context
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: *2153806060
+- !ruby/object:Gem::Dependency
+  name: shoulda-context
+  requirement: &2153805640 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id004
+  prerelease: false
+  version_requirements: *2153805640
 description: Sending haproxy logs to new relic rpm
-email: 
+email:
 - patrick.huesler@gmail.com
-executables: 
+executables:
 - haproxy2rpm
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
 - Gemfile
 - README.md
 - Rakefile
+- Research.md
 - bin/haproxy2rpm
 - haproxy2rpm.gemspec
 - lib/haproxy2rpm.rb
@@ -79,38 +77,33 @@ files:
 - lib/haproxy2rpm/rpm.rb
 - lib/haproxy2rpm/syslog.rb
 - lib/haproxy2rpm/version.rb
-- test/fixtures/haproxy.log
 - test/haproxy2pm_test.rb
 - test/test_helper.rb
 has_rdoc: true
-homepage: ""
+homepage: ''
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
 rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Sending haproxy logs to new relic rpm
-test_files: 
-- test/fixtures/haproxy.log
+test_files:
 - test/haproxy2pm_test.rb
 - test/test_helper.rb

data/test/fixtures/haproxy.log

@@ -1 +0,0 @@
-haproxy[674]: 127.0.0.1:33319 [15/Oct/2003:08:31:57] relais-http Srv1 6559/0/7/147/6723 200 243 - - ---- 1/3/5 0/0 "HEAD / HTTP/1.0"