logjam_agent 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in logjam_agent.gemspec
+gemspec

data/README.rdoc

@@ -0,0 +1,8 @@
+== Logjam Agent
+
+Client side library for logjam.
+
+Provides a customized logger, which logs lines and sends combined request information via
+AMQP to logjam.
+
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/init.rb

@@ -0,0 +1 @@
+ActionController::Dispatcher.middleware.insert_before(ActionController::Failsafe, LogjamAgent::Middleware)

data/lib/logjam_agent.rb

@@ -0,0 +1,18 @@
+require "logjam_agent/version"
+require "logjam_agent/amqp_forwarder"
+require "logjam_agent/forwarders"
+require "logjam_agent/request"
+require "logjam_agent/buffered_logger"
+require "logjam_agent/syslog_like_formatter"
+
+module LogjamAgent
+
+  class ForwardingError < StandardError; end
+
+  mattr_accessor :error_handler
+  self.error_handler = lambda { |exception| }
+
+  mattr_accessor :application_name
+  self.application_name = "rails"
+
+end

data/lib/logjam_agent/amqp_forwarder.rb

@@ -0,0 +1,94 @@
+module LogjamAgent
+  class AMQPForwarder
+
+    RETRY_AFTER = 10.seconds
+
+    attr_reader :app, :env
+
+    def initialize(app, env, opts = {})
+      @app = app
+      @env = env
+      @config = default_options(app, env).merge!(opts)
+      @exchange = @bunny = nil
+    end
+
+    def default_options(app, env)
+      {
+        :host                 => "localhost",
+        :exchange             => "request-stream-#{app}-#{env}",
+        :exchange_durable     => true,
+        :exchange_auto_delete => false,
+        :routing_key          => "logs.#{app}.#{env}"
+      }
+    end
+
+    # TODO: mutex!
+    def send(msg)
+      return if paused?
+      begin
+        # $stderr.puts msg
+        exchange.publish(msg, :key => @config[:routing_key], :persistent => false)
+      rescue Exception => exception
+        reraise_expectation_errors!
+        pause(exception)
+      end
+    end
+
+    def reset(exception=nil)
+      return unless @bunny
+      begin
+        if exception
+          @bunny.__send__(:close_socket)
+        else
+          @bunny.stop
+        end
+      rescue Exception
+        # if bunny throws an exception here, its not usable anymore anyway
+      ensure
+        @exchange = @bunny = nil
+      end
+    end
+
+    private
+
+    if defined?(Mocha)
+      def reraise_expectation_errors! #:nodoc:
+        raise if $!.is_a?(Mocha::ExpectationError)
+      end
+    else
+      def reraise_expectation_errors! #:nodoc:
+        # noop
+      end
+    end
+
+    def pause(exception)
+      @paused = Time.now
+      reset(exception)
+      raise ForwardingError.new("Could not log to AMQP exchange (#{exception.message})")
+    end
+
+    def paused?
+      @paused && @paused > RETRY_AFTER.ago
+    end
+
+    def exchange
+      @exchange ||=
+        begin
+          bunny.start unless bunny.connected?
+          bunny.exchange(@config[:exchange],
+                         :durable => @config[:exchange_durable],
+                         :auto_delete => @config[:exchange_auto_delete],
+                         :type => :topic)
+        end
+    end
+
+    def bunny
+      @bunny ||=
+        begin
+          require "bunny" unless defined?(Bunny)
+          Bunny.new(:host => @config[:host], :socket_timeout => 1.0)
+        end
+    end
+
+  end
+end

data/lib/logjam_agent/buffered_logger.rb

@@ -0,0 +1,55 @@
+require 'active_support/buffered_logger'
+require 'active_support/core_ext/logger'
+require 'fileutils'
+
+module LogjamAgent
+  class BufferedLogger < ActiveSupport::BufferedLogger
+
+    attr_accessor :formatter
+
+    def initialize(*args)
+      super(*args)
+      # stupid bug in the buffered logger code
+      @log.write "\n"
+      @formatter = lambda{|_, _, _, message| message}
+    end
+
+    def start_request(app, env, initial_fields={})
+      Thread.current[:logjam_request] = Request.new(app, env, self, initial_fields)
+    end
+
+    def finish_request(additional_fields={})
+      # puts "finishing request"
+      if request = self.request
+        request.fields.merge!(additional_fields)
+        Thread.current[:logjam_request] = nil
+        request.forward
+      end
+    end
+
+    def request
+      Thread.current[:logjam_request]
+    end
+
+    def add(severity, message = nil, progname = nil, &block)
+      return if @level > severity
+      message = (message || (block && block.call) || '').to_s
+      # If a newline is necessary then create a new message ending with a newline.
+      # Ensures that the original message is not mutated.
+      message = "#{message}\n" unless message[-1] == ?\n
+      time = Time.now
+      buffer << formatter.call(severity, time, progname, message)
+      auto_flush
+      if request = self.request
+        # puts "adding line to request"
+        request.add_line(severity, time, message[0..-2])
+      end
+      message
+    end
+
+    def logdev=(log_device)
+      raise "cannot connect logger to new log device" unless log_device.respond_to?(:write)
+      @log = log_device
+    end
+  end
+end

data/lib/logjam_agent/forwarders.rb

@@ -0,0 +1,19 @@
+module LogjamAgent
+  module Forwarders
+    @@forwarders = {}
+
+    extend self
+
+    def add(f)
+      @@forwarders["#{f.app}-#{f.env}"] = f
+    end
+
+    def get(app, env)
+      @@forwarders["#{app}-#{env}"]
+    end
+
+    def reset
+      @@forwarders.each_value {|f| f.reset}
+    end
+  end
+end

data/lib/logjam_agent/middleware.rb

@@ -0,0 +1,25 @@
+module LogjamAgent
+  class Middleware
+    def initialize(app, options={})
+      @app = app
+      @options = options
+    end
+
+    def call(env)
+      start_request(env)
+      @app.call(env)
+    ensure
+      finish_request(env)
+    end
+
+    private
+
+    def start_request(env)
+      Rails.logger.start_request(env["logjam_agent.application_name"]||LogjamAgent.application_name, Rails.env)
+    end
+
+    def finish_request(env)
+      Rails.logger.finish_request(env["time_bandits.metrics"])
+    end
+  end
+end

data/lib/logjam_agent/request.rb

@@ -0,0 +1,42 @@
+require "json"
+require "socket"
+
+module LogjamAgent
+  class Request
+    attr_reader :fields
+
+    @@hostname = Socket.gethostname.split('.').first
+
+    def initialize(app, env, logger, initial_fields)
+      @logger = logger
+      @forwarder = Forwarders.get(app, env)
+      @lines = []
+      @fields = initial_fields.merge(:host => @@hostname, :process_id => Process.pid, :lines => @lines)
+    end
+
+    def add_line(severity, timestamp, message)
+      @lines << [severity, format_time(timestamp), message]
+    end
+
+    def forward
+      @forwarder.send(@fields.to_json)
+    rescue Exception => e
+      handle_forwarding_error(e)
+    end
+
+    private
+
+    def format_time(t)
+      # iso time with microseconds
+      t.strftime("%Y-%m-%dT%H:%M:%S.#{t.usec}")
+    end
+
+    def handle_forwarding_error(exception)
+      @logger.error exception.to_s
+      LogjamAgent.error_handler.call(exception)
+    rescue Exception
+      # swallow all exceptions
+    end
+
+  end
+end

data/lib/logjam_agent/syslog_like_formatter.rb

@@ -0,0 +1,39 @@
+require 'socket'
+require 'logger'
+
+module LogjamAgent
+  class SyslogLikeFormatter
+    def initialize
+      @hostname = Socket.gethostname.split('.').first
+      @app_name = "rails"
+    end
+
+    attr_accessor :extra_attributes
+
+    SEV_LABEL = Logger::SEV_LABEL.map{|sev| "%-5s" % sev}
+
+    def format_severity(severity)
+      SEV_LABEL[severity] || 'ALIEN'
+    end
+
+    def format_time(timestamp)
+      timestamp.strftime("%b %d %H:%M:%S.#{timestamp.usec}")
+    end
+
+    def format_msg(msg)
+      "#{msg}".sub(/^[\s\n]+/, '').sub(/[\s\n]+$/,"\n")
+    end
+
+    def call(severity, timestamp, progname, msg)
+      "#{format_severity(severity)} #{format_time(timestamp)} #{@hostname} #{progname||@app_name}[#{$$}]#{render_extra_attributes}: #{format_msg(msg)}"
+    end
+
+    def render_extra_attributes
+      (self.extra_attributes||[]).map{|key, value| " #{key}[#{value}]"}
+    end
+
+    def add_extra_attributes(attributes)
+      (self.extra_attributes ||= []).concat(attributes)
+    end
+  end
+end

data/lib/logjam_agent/version.rb

@@ -0,0 +1,3 @@
+module LogjamAgent
+  VERSION = "0.0.1"
+end

data/logjam_agent.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "logjam_agent/version"
+
+Gem::Specification.new do |s|
+  s.name        = "logjam_agent"
+  s.version     = LogjamAgent::VERSION
+  s.authors     = ["Stefan Kaes"]
+  s.email       = ["stefan.kaes@xing.com"]
+  s.homepage    = ""
+  s.summary     = %q{Logjam client library to be used with logjam}
+  s.description = %q{Logjam logger and request information forwarding}
+
+  s.rubyforge_project = "logjam_agent"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  s.add_runtime_dependency "activesupport"
+  s.add_runtime_dependency "time_bandits", [">= 0.1.0"]
+end

data/rails/init.rb

@@ -0,0 +1 @@
+require File.expand_path('../../init.rb', __FILE__)

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification 
+name: logjam_agent
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Stefan Kaes
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-28 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: time_bandits
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 0
+        - 1
+        - 0
+        version: 0.1.0
+  type: :runtime
+  version_requirements: *id002
+description: Logjam logger and request information forwarding
+email: 
+- stefan.kaes@xing.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- README.rdoc
+- Rakefile
+- init.rb
+- lib/logjam_agent.rb
+- lib/logjam_agent/amqp_forwarder.rb
+- lib/logjam_agent/buffered_logger.rb
+- lib/logjam_agent/forwarders.rb
+- lib/logjam_agent/middleware.rb
+- lib/logjam_agent/request.rb
+- lib/logjam_agent/syslog_like_formatter.rb
+- lib/logjam_agent/version.rb
+- logjam_agent.gemspec
+- rails/init.rb
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: logjam_agent
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Logjam client library to be used with logjam
+test_files: []
+