termite 0.0.10 → 0.0.20

data/lib/termite/hastur_logger.rb

@@ -0,0 +1,41 @@
+module Termite
+  class HasturLogger
+    def initialize(socket, addr, port, labels)
+      @socket, @addr, @port = socket, addr, port
+      @labels = labels || {}
+    end
+
+    def to_usec(time)
+      (time.to_f * 1_000_000).round
+    end
+
+    def send_message(severity, raw_message, app_data, time=Time.now, data='{}')
+      severity = Logger::LOGGER_LEVEL_MAP.invert[severity].to_s
+      tid = Ecology.thread_id(::Thread.current)
+      hostname = Socket.gethostname
+      pid = Process.pid
+      application, component = app_data[:app], app_data[:component]
+
+      message = {
+        :_route => :log,
+        :timestamp => to_usec(time),
+        :message => raw_message
+      }
+
+      labels = {
+        :severity => severity,
+        :pid => pid,
+        :tid => tid,
+        :app => application,
+        :component => component,
+        :hostname => hostname
+      }
+
+      labels.merge!(MultiJson.decode(data)) if data && data != '{}'
+      labels.merge!(@labels) if @labels && !@labels.empty?
+      message[:labels] = labels
+
+      @socket.send MultiJson.encode(message), 0, @addr, @port
+    end
+  end
+end

data/lib/termite/syslog_logger.rb

@@ -0,0 +1,47 @@
+require "syslog"
+
+module Termite
+  class SyslogLogger
+    def initialize(socket, server_addr, server_port, transport)
+      @socket, @server_addr, @server_port, @transport = socket, server_addr, server_port, transport
+    end
+
+    def send_message(severity, full_message, app_data, time=Time.now, data='{}')
+      tid = Ecology.thread_id(::Thread.current)
+      day = time.strftime("%b %d").sub(/0(\d)/, ' \\1')
+      time_of_day = time.strftime("%T")
+      hostname = Socket.gethostname
+      application = app_data[:combined] || app_data[:app]
+
+      # Convert Ruby log level to syslog severity
+      tag = Syslog::LOG_LOCAL6 + Logger::SYSLOG_SEVERITY_MAP[Logger::LEVEL_SYSLOG_MAP[severity]]
+      syslog_string = "<#{tag}>#{day} #{time_of_day} #{hostname} #{application} [#{Process.pid}]: [#{tid}] "
+
+      # ruby_severity is the Ruby Logger severity as a symbol
+      ruby_severity = Logger::LOGGER_LEVEL_MAP.invert[severity]
+
+      full_message.split("\n").each do |line|
+        syslog_message = "#{line} #{data}"
+        case @transport
+        when "UDP"
+          send_udp(severity, syslog_string, syslog_message, application)
+        when "syscall", "libc"
+          send_libc(severity, syslog_message, application) rescue nil
+        else
+          send_libc(severity, syslog_message, application) rescue
+            send_udp(severity, syslog_string, syslog_message, application)
+        end
+      end
+    end
+
+    def send_udp(severity, syslog_string, syslog_message, application)
+        @socket.send(syslog_string + syslog_message, 0, @server_addr, @server_port) rescue nil
+    end
+
+    def send_libc(severity, syslog_message, application)
+      Syslog.open(application, Syslog::LOG_PID | Syslog::LOG_CONS) do |s|
+        s.send(Logger::LEVEL_SYSLOG_MAP[severity], syslog_message)
+      end
+    end
+  end
+end

data/lib/termite/version.rb

@@ -1,3 +1,3 @@
 module Termite
-  VERSION = "0.0.10"
+  VERSION = "0.0.20"
 end

data/lib/termite/version.rb~

@@ -0,0 +1,3 @@
+module Termite
+  VERSION = "0.0.14"
+end

data/termite.gemspec

@@ -25,8 +25,9 @@ EOS
 
   s.require_paths = ["lib"]
 
-  s.add_dependency "multi_json"
   s.add_dependency "ecology", "~>0.0.6"
+  s.add_dependency "multi_json"
+  s.add_dependency "rainbow", "~> 1.1.3"
 
   s.add_development_dependency "bundler", "~> 1.0.10"
   s.add_development_dependency "scope", "~> 0.2.1"

data/test/ecology_log_test.rb

@@ -29,17 +29,17 @@ ECOLOGY_CONTENTS
       end
 
       should "send back extra JSON data and a default component when specified" do
-        expect_add(@logger.socket, 2, 'oh no! {"app_group":"SuperSpiffyGroup","precedence":7}', :application => "MyApp:SplodgingLib")
+        expect_add(2, 'oh no! {"app_group":"SuperSpiffyGroup","precedence":7}', :application => "MyApp:SplodgingLib")
         @logger.fatal("oh no!")
       end
 
       should "allow overriding the default component" do
-        expect_add(@logger.socket, 2, 'oh no! {"app_group":"SuperSpiffyGroup","precedence":7}', :application => "MyApp:SpliyingLib")
+        expect_add(2, 'oh no! {"app_group":"SuperSpiffyGroup","precedence":7}', :application => "MyApp:SpliyingLib")
         @logger.fatal("oh no!", {}, :component => "SpliyingLib")
       end
 
       should "allow overriding the default component with nothing" do
-        expect_add(@logger.socket, 2, 'oh no! {"app_group":"SuperSpiffyGroup","precedence":7}', :application => "MyApp")
+        expect_add(2, 'oh no! {"app_group":"SuperSpiffyGroup","precedence":7}', :application => "MyApp")
         @logger.fatal("oh no!", {}, :component => nil)
       end
     end
@@ -53,6 +53,7 @@ ECOLOGY_CONTENTS
     should "override parameters passed to Termite Logger" do
       log_mock = mock("Ruby Logger")
       ::Logger.expects(:new).with("/tmp/bobo.txt", 10, 1024000).returns(log_mock)
+      ::Logger.expects(:new).with("/var/lib/sam.log", "daily", 1048576).returns(log_mock)
       Termite::Logger.new("/var/lib/sam.log", "daily")
     end
   end

data/test/extra_logger_test.rb

@@ -11,15 +11,16 @@ class TermiteExtraLoggerTest < Scope::TestCase
       setup do
         @logger = Termite::Logger.new("/tmp/test_log_output.txt")  # Test with output file
         @logger.level = Logger::DEBUG
-        @logger.socket.expects(:send)
+        Syslog.expects(:open)
         @mock_logger_1 = mock()
+        @mock_logger_2 = mock()
         @logger.add_logger(@mock_logger_1)
         @logger.add_logger(@mock_logger_2)
       end
 
-      should "correctly send logs to Syslog" do
-        @mock_logger_1.expects(:fatal)
-        @mock_logger_2.expects(:fatal)
+      should "correctly send logs to additional loggers" do
+        @mock_logger_1.expects(:<<).with("foo!")
+        @mock_logger_2.expects(:<<).with("foo!")
         @logger.add(Logger::FATAL, "foo!", {})
       end
     end

data/test/hastur_logger_test.rb

@@ -0,0 +1,58 @@
+require File.join(File.dirname(__FILE__), "test_helper.rb")
+
+class HasturLoggerTest < Scope::TestCase
+
+  def expect_send(raw_message, data)
+    message = {
+      :_route => :log,
+      :timestamp => @time_usec,
+      :message => raw_message
+    }
+
+    labels = {
+      :severity => "warn",
+      :pid => @pid,
+      :tid => @tid,
+      :app => @app_data[:app],
+      :component => @app_data[:component],
+      :hostname => @host
+    }
+
+    message[:labels] = labels.merge(MultiJson.decode(data)).merge(@labels)
+    json_message = MultiJson.encode(message)
+
+    @socket.expects(:send).with(json_message, 0, @addr, @port)
+  end
+
+  context "with a hastur logger" do
+    setup do
+      @socket = mock("UDP Socket")
+      @labels = {"hi" => "lo", "fast" => "slo"}
+      @addr = "127.0.0.1"
+      @port = 8125
+      @logger = Termite::HasturLogger.new(@socket, @addr, @port, @labels)
+    end
+
+    context "will send a message" do
+      setup do
+        @severity = 2
+        @raw_message = "message"
+        @app_data = {:combined => "app:comp", :app => "app", :component => "comp"}
+        @pid = 123
+        @tid = "main"
+        @time = 345
+        @time_usec = 345_000_000
+        @host = "host"
+        Ecology.stubs(:thread_id).returns(@tid)
+        Process.stubs(:pid).returns(@pid)
+        Time.stubs(:now).returns(@time)
+        Socket.stubs(:gethostname).returns(@host)
+      end
+
+      should "send back extra JSON data and labels when specified" do
+        expect_send("oh no!", '{"app_group":"SuperSpiffyGroup","precedence":7}')
+        @logger.send_message(@severity, "oh no!", @app_data, Time.now, '{"app_group":"SuperSpiffyGroup","precedence":7}')
+      end
+    end
+  end
+end

data/test/level_test.rb

@@ -0,0 +1,53 @@
+require File.join(File.dirname(__FILE__), "test_helper.rb")
+
+class LevelTest < Scope::TestCase
+  context "without termite ecology" do
+    setup do
+      @logger = Termite::Logger.new(:level => "error")
+    end
+
+    should "correctly set level to error" do
+      assert_equal Logger::ERROR, @logger.level
+    end
+  end
+
+  context "with termite ecology" do
+    setup do
+      Ecology.reset
+
+      set_up_ecology <<ECOLOGY_TEXT
+{
+  "application": "foo_app",
+  "logging":
+    {
+      "default_component": "whatcomponent",
+      "level": "info"
+    }
+}
+ECOLOGY_TEXT
+    end
+
+    context "and no overrides" do
+      setup do
+        @logger = Termite::Logger.new
+      end
+
+      should "correctly have level set at info" do
+        assert_equal Logger::INFO, @logger.level
+      end
+    end
+
+    context "and debug overriding at env level" do
+      setup do
+        ENV["TERMITE_DEBUG"] = "1"
+        @logger = Termite::Logger.new()
+      end
+
+      should "correctly have level set at debug" do
+        assert_equal Logger::DEBUG, @logger.level
+      end
+
+    end
+
+  end
+end

data/test/rescue_test.rb

@@ -9,34 +9,32 @@ class RescueTest < Scope::TestCase
     end
 
     should "continue even if all loggers raise errors" do
-      # First, UDP will raise an error
-      @logger.socket.expects(:send).raises(StandardError, "You suck!")
-
-      # Termite should fall back to trying Ruby Syslog...
+      # First, termite should try Ruby Syslog...
       syslog_mock = mock("Syslog connection")
       Syslog.expects(:open).yields(syslog_mock)
-      syslog_mock.expects(:error).with("UDP syslog failed!  Falling back to libc syslog!")
       syslog_mock.expects(:crit).raises(StandardError, "You suck even more than that!")
 
-      # And it should still try to write to a file logger
-      @logger.file_logger.expects(:fatal).raises(StandardError, "You suck lots!")
+      # Then, it will fal back to UDP and raise an error
+      @logger.socket.expects(:send).raises(StandardError, "You suck!")
+
+      # And it should still try to write to a file logger - this is now just an extra logger
+      # @logger.file_logger.expects(:fatal).raises(StandardError, "You suck lots!")
 
       extra_logger = mock("Additional logger")
       @logger.add_logger(extra_logger)
       # And it should try to write to any extra loggers
-      extra_logger.expects(:fatal).raises(StandardError, "You suck constantly!")
+      extra_logger.expects(:<<).raises(StandardError, "You suck constantly!")
 
       # And yet, nothing should be raised to the outside world
       begin
         @logger.fatal("Woe is me!")
         assert true, "Nothing was raised!  Yay!"
       rescue Exception
-        flunk "Logging an event raised an assertion outside the logger!"        
+        flunk "Logging an event raised an assertion outside the logger!"
       end
     end
 
     should "continue even if internal logic gives an error" do
-      #Time.expects(:now).raises(Exception.new "You suck!")
       Ecology.expects(:thread_id).raises(Exception.new "Ecology thread_id dies!")
       begin
         @logger.fatal("Woe is me!")

data/test/sinks_test.rb

@@ -0,0 +1,183 @@
+require File.join(File.dirname(__FILE__), "test_helper.rb")
+
+class SinksTest < Scope::TestCase
+  context "with a hastur sink ecology" do
+    setup do
+      Ecology.reset
+
+      set_up_ecology <<ECOLOGY_CONTENTS
+{
+  "application": "MyApp",
+  "logging": {
+    "default_component": "SplodgingLib",
+    "extra_json_fields": {
+      "app_group": "SuperSpiffyGroup",
+      "precedence": 7
+    },
+    "console_print": "off",
+    "sinks": [
+      {
+        "type": "syslog",
+        "transport": "UDP"
+      },
+      {
+        "type": "hastur",
+        "labels": {
+          "hi": "lo",
+          "fast": "slo"
+        }
+      }
+    ]
+  }
+}
+ECOLOGY_CONTENTS
+    end
+
+    context "with a default termite logger" do
+      setup do
+        @hastur_mock = mock("Hastur Logger")
+        Termite::HasturLogger.expects(:new).returns(@hastur_mock)
+        @logger = Termite::Logger.new
+      end
+
+      should "send back extra JSON data and a default component when specified" do
+        @hastur_mock.expects(:send_message).with(4, 'oh no!', {:combined => "MyApp:SplodgingLib", :app => "MyApp", :component => "SplodgingLib"}, anything, '{"app_group":"SuperSpiffyGroup","precedence":7}')
+        @logger.fatal("oh no!")
+      end
+
+      should "allow overriding the default component" do
+        @hastur_mock.expects(:send_message).with(4, 'oh no!', {:combined => "MyApp:SpliyingLib", :app => "MyApp", :component => "SpliyingLib"}, anything, '{"app_group":"SuperSpiffyGroup","precedence":7}')
+        @logger.fatal("oh no!", {}, :component => "SpliyingLib")
+      end
+
+      should "allow overriding the default component with nothing" do
+        @hastur_mock.expects(:send_message).with(4, 'oh no!', {:combined => nil, :app => "MyApp", :component => nil}, anything, '{"app_group":"SuperSpiffyGroup","precedence":7}')
+        @logger.fatal("oh no!", {}, :component => nil)
+      end
+    end
+  end
+
+  context "with a sinked UDP ecology" do
+    setup do
+      Ecology.reset
+
+      set_up_ecology <<ECOLOGY_CONTENTS
+{
+  "application": "MyApp",
+  "logging": {
+    "default_component": "SplodgingLib",
+    "extra_json_fields": {
+      "app_group": "SuperSpiffyGroup",
+      "precedence": 7
+    },
+    "console_print": "off",
+    "sinks": [
+      {
+        "type": "syslog",
+        "transport": "UDP"
+      }
+    ]
+  }
+}
+ECOLOGY_CONTENTS
+    end
+
+    context "with a default termite logger" do
+      setup do
+        @logger = Termite::Logger.new
+      end
+
+      should "send back extra JSON data and a default component when specified" do
+        expect_udp(@logger.socket, 2, 'oh no! {"app_group":"SuperSpiffyGroup","precedence":7}', :application => "MyApp:SplodgingLib")
+        @logger.fatal("oh no!")
+      end
+
+      should "allow overriding the default component" do
+        expect_udp(@logger.socket, 2, 'oh no! {"app_group":"SuperSpiffyGroup","precedence":7}', :application => "MyApp:SpliyingLib")
+        @logger.fatal("oh no!", {}, :component => "SpliyingLib")
+      end
+
+      should "allow overriding the default component with nothing" do
+        expect_udp(@logger.socket, 2, 'oh no! {"app_group":"SuperSpiffyGroup","precedence":7}', :application => "MyApp")
+        @logger.fatal("oh no!", {}, :component => nil)
+      end
+    end
+  end
+
+  context "with a sinked ecology" do
+    setup do
+      Ecology.reset
+
+      set_up_ecology <<ECOLOGY_CONTENTS
+{
+  "application": "MyApp",
+  "logging": {
+    "default_component": "SplodgingLib",
+    "extra_json_fields": {
+      "app_group": "SuperSpiffyGroup",
+      "precedence": 7
+    },
+    "console_print": "off",
+    "sinks": [
+      {
+        "type": "stdout",
+        "color": "green",
+        "min_level": "debug",
+        "max_level": "warn"
+      },
+      {
+        "type": "stderr",
+        "color": "red",
+        "min_level": "error"
+      },
+      {
+        "type": "file",
+        "filename": "/tmp/bobo.txt",
+        "min_level": "warn",
+        "shift_age": 10,
+        "shift_size": 1024000
+      },
+      {
+        "type": "syslog"
+      }
+    ]
+  }
+}
+ECOLOGY_CONTENTS
+    end
+
+    context "with a default termite logger" do
+      setup do
+        @logger = Termite::Logger.new
+      end
+
+      should "send back extra JSON data and a default component when specified" do
+        expect_add(2, 'oh no! {"app_group":"SuperSpiffyGroup","precedence":7}', :application => "MyApp:SplodgingLib")
+        @logger.fatal("oh no!")
+      end
+
+      should "allow overriding the default component" do
+        expect_add(2, 'oh no! {"app_group":"SuperSpiffyGroup","precedence":7}', :application => "MyApp:SpliyingLib")
+        @logger.fatal("oh no!", {}, :component => "SpliyingLib")
+      end
+
+      should "allow overriding the default component with nothing" do
+        expect_add(2, 'oh no! {"app_group":"SuperSpiffyGroup","precedence":7}', :application => "MyApp")
+        @logger.fatal("oh no!", {}, :component => nil)
+      end
+    end
+
+    should "pass initialize parameters to Ruby Logger" do
+      log_mock = mock("Ruby Logger")
+      ::Logger.expects(:new).with("/tmp/bobo.txt", 10, 1024000).returns(log_mock)
+      Termite::Logger.new
+    end
+
+    should "override parameters passed to Termite Logger" do
+      log_mock = mock("Ruby Logger")
+      ::Logger.expects(:new).with("/tmp/bobo.txt", 10, 1024000).returns(log_mock)
+      ::Logger.expects(:new).with("/var/lib/sam.log", "daily", 1048576).returns(log_mock)
+      Termite::Logger.new("/var/lib/sam.log", "daily")
+    end
+  end
+end