logstash-logger 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 00421e5706d99d473e29844c80e2c50f471f2ca1
-  data.tar.gz: a1ae4bd4936c8acecbc560391bda1e70f624ecb8
+  metadata.gz: 1b5f101aba35d73c9bb62e77c6a5be82321c54de
+  data.tar.gz: 85d7303ebb637bda83b42dec2794c7551983e22e
 SHA512:
-  metadata.gz: 194c6aec69634dfd22112a120fa6c9dc5e459f82eae2e97f78b1e512907bb20d7f475edb7d60f66ffec6637a72cafd5ba9c1079ae6bf6bf83696aa2276570f00
-  data.tar.gz: 495dfc7623df42e62418e20b274185bb2c1e40fa588a166fd09af97dc5878d4cf721aaaa463df7c6a93706a1aed8da49057ce00074fdf9b6cfb3b4a1a24fd67c
+  metadata.gz: 0ce2d236f92070efa47bf40cbc708a2fa3c12d65f1e611813b6170678ce4b00e5889661f0e9b9d3ce7760f0f6eef12ea1773eb9ac98b843f8ae57f0514d4463f
+  data.tar.gz: 5c038e89a27a7467a76da3a7715d2161ca5afdf3f1bd6ec25216c433aaedd979399cbc0d133dd96b20760970637543814ddea4f80314612df174d915f369f3d7

data/.travis.yml

@@ -14,3 +14,4 @@ gemfile:
 matrix:
   allow_failures:
     - rbx-2
+sudo: false

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.12.0
+- Support for other formatters, including custom formatters. [#56](https://github.com/dwbutler/logstash-logger/pull/56)
+- Support for CEE output.
+
 ## 0.11.0
 - Support for balancer log device. [#55](https://github.com/dwbutler/logstash-logger/pull/55)
 

data/README.md

@@ -28,7 +28,7 @@ Or install it yourself as:
 
     $ gem install logstash-logger
 
-## Basic Usage
+## Usage Examples
 
 ```ruby
 require 'logstash-logger'
@@ -49,6 +49,12 @@ stdout_logger = LogStashLogger.new(type: :stdout)
 stderr_logger = LogStashLogger.new(type: :stderr)
 io_logger = LogStashLogger.new(type: :io, io: io)
 
+# Use a different formatter
+cee_logger = LogStashLogger.new(type: :tcp, host: 'logsene-receiver-syslog.sematext.com', port: 514, formatter: :cee_syslog)
+custom_formatted_logger = LogStashLogger.new(type: :redis, formatter: MyCustomFormatter)
+lambda_formatted_logger = LogStashLogger.new(type: :stdout, formatter: ->(severity, time, progname, msg) { "[#{progname}] #{msg}" })
+ruby_default_formatter_logger = LogStashLogger.new(type: :file, path: 'log/development.log', formatter: ::Logger::Formatter)
+
 # Multiple Outputs
 multi_logger = LogStashLogger.new([{type: :file, path: 'log/development.log'}, {type: :udp, host: 'localhost', port: 5228}])
 
@@ -77,7 +83,7 @@ logger.tagged('foo') { logger.fatal('bar') }
 ## URI Configuration
 You can use a URI to configure your logstash logger instead of a hash. This is useful in environments
 such as Heroku where you may want to read configuration values from the environment. The URI scheme
-is `type://host:port/path`. Some sample URI configurations are given below.
+is `type://host:port/path?key=value`. Some sample URI configurations are given below.
 
 ```
 udp://localhost:5228
@@ -97,7 +103,7 @@ Pass the URI into your logstash logger like so:
 logger = LogStashLogger.new(uri: ENV['LOGSTASH_URI'])
 ```
 
-## Logstash Configuration
+## Logstash Listener Configuration
 
 In order for logstash to correctly receive and parse the events, you will need to
 configure and run a listener that uses the `json_lines` codec. For example, to receive

data/lib/logstash-logger/formatter.rb

@@ -1,51 +1,48 @@
-require 'logger'
-require 'socket'
-require 'time'
+require 'logstash-logger/formatter/base'
 
 module LogStashLogger
-  HOST = ::Socket.gethostname
+	module Formatter
+		DEFAULT_FORMATTER = :json_lines
 
-  class Formatter < ::Logger::Formatter
-    include TaggedLogging::Formatter
+    autoload :LogStashEvent, 'logstash-logger/formatter/logstash_event'
+    autoload :Json, 'logstash-logger/formatter/json'
+    autoload :JsonLines, 'logstash-logger/formatter/json_lines'
+    autoload :Cee, 'logstash-logger/formatter/cee'
+    autoload :CeeSyslog, 'logstash-logger/formatter/cee_syslog'
 
-    def call(severity, time, progname, message)
-      event = build_event(message, severity, time)
-      "#{event.to_json}\n"
+		def self.new(formatter_type)
+    	build_formatter(formatter_type)
     end
 
-    protected
+    def self.build_formatter(formatter_type)
+      formatter_type ||= DEFAULT_FORMATTER
 
-    def build_event(message, severity, time)
-      data = message
-      if data.is_a?(String) && data.start_with?('{')
-        data = (JSON.parse(message) rescue nil) || message
+      if custom_formatter_instance?(formatter_type)
+        formatter_type
+      elsif custom_formatter_class?(formatter_type)
+        formatter_type.new
+      else
+        formatter_klass(formatter_type).new
       end
+    end
 
-      event = case data
-                when LogStash::Event
-                  data.clone
-                when Hash
-                  event_data = data.merge("@timestamp" => time)
-                  LogStash::Event.new(event_data)
-                else
-                  LogStash::Event.new("message" => msg2str(data), "@timestamp" => time)
-              end
-
-      event['severity'] ||= severity
-      #event.type = progname
-
-      event['host'] ||= HOST
+    def self.formatter_klass(formatter_type)
+      case formatter_type.to_sym
+      when :json_lines then JsonLines
+      when :json then Json
+      when :logstash_event then LogStashEvent
+      when :cee then Cee
+      when :cee_syslog then CeeSyslog
+      else fail ArgumentError, 'Invalid formatter'
+      end
+    end
 
-      current_tags.each { |tag| event.tag(tag) }
-      
-      LogStashLogger.configuration.customize_event_block.call(event) if LogStashLogger.configuration.customize_event_block.respond_to?(:call)
+    def self.custom_formatter_instance?(formatter_type)
+      formatter_type.respond_to?(:call)
+    end
 
-      # In case Time#to_json has been overridden
-      if event.timestamp.is_a?(Time)
-        event.timestamp = event.timestamp.iso8601(3)
-      end
-      
-      event
+    def self.custom_formatter_class?(formatter_type)
+      formatter_type.is_a?(Class) && formatter_type.method_defined?(:call)
     end
-  end
+	end
 end

data/lib/logstash-logger/formatter/base.rb

@@ -0,0 +1,52 @@
+require 'logger'
+require 'socket'
+require 'time'
+
+module LogStashLogger
+  module Formatter
+    HOST = ::Socket.gethostname
+
+    class Base < ::Logger::Formatter
+      include ::LogStashLogger::TaggedLogging::Formatter
+
+      def call(severity, time, progname, message)
+        @event = build_event(message, severity, time)
+      end
+
+      protected
+
+      def build_event(message, severity, time)
+        data = message
+        if data.is_a?(String) && data.start_with?('{'.freeze)
+          data = (JSON.parse(message) rescue nil) || message
+        end
+
+        event = case data
+                  when LogStash::Event
+                    data.clone
+                  when Hash
+                    event_data = data.merge("@timestamp".freeze => time)
+                    LogStash::Event.new(event_data)
+                  else
+                    LogStash::Event.new("message".freeze => msg2str(data), "@timestamp".freeze => time)
+                end
+
+        event['severity'.freeze] ||= severity
+        #event.type = progname
+
+        event['host'.freeze] ||= HOST
+
+        current_tags.each { |tag| event.tag(tag) }
+        
+        LogStashLogger.configuration.customize_event_block.call(event) if LogStashLogger.configuration.customize_event_block.respond_to?(:call)
+
+        # In case Time#to_json has been overridden
+        if event.timestamp.is_a?(Time)
+          event.timestamp = event.timestamp.iso8601(3)
+        end
+        
+        event
+      end
+    end
+  end
+end

data/lib/logstash-logger/formatter/cee.rb

@@ -0,0 +1,10 @@
+module LogStashLogger
+  module Formatter
+    class Cee < Base
+      def call(severity, time, progname, message)
+        super
+        "@cee:#{@event.to_json}"
+      end
+    end
+  end
+end

data/lib/logstash-logger/formatter/cee_syslog.rb

@@ -0,0 +1,20 @@
+module LogStashLogger
+  module Formatter
+    class CeeSyslog < Cee
+      def call(severity, time, progname, message)
+        @cee = super
+        @progname = progname
+
+        "#{facility}:#{@cee}\n"
+      end
+
+      protected
+
+      def facility
+        @facility = "#{@event['host']}"
+        @facility << " #{@progname}" if @progname
+        @facility
+      end
+    end
+  end
+end

data/lib/logstash-logger/formatter/json.rb

@@ -0,0 +1,10 @@
+module LogStashLogger
+  module Formatter
+    class Json < Base
+      def call(severity, time, progname, message)
+        super
+        @event.to_json
+      end
+    end
+  end
+end

data/lib/logstash-logger/formatter/json_lines.rb

@@ -0,0 +1,10 @@
+module LogStashLogger
+  module Formatter
+    class JsonLines < Base
+      def call(severity, time, progname, message)
+        super
+        "#{@event.to_json}\n"
+      end
+    end
+  end
+end

data/lib/logstash-logger/formatter/logstash_event.rb

@@ -0,0 +1,9 @@
+module LogStashLogger
+  module Formatter
+    class LogStashEvent < Base
+      def call(severity, time, progname, message)
+        super
+      end
+    end
+  end
+end

data/lib/logstash-logger/logger.rb

@@ -4,13 +4,14 @@ require 'logstash-logger/tagged_logging'
 module LogStashLogger
   def self.new(*args)
     opts = extract_opts(*args)
+    formatter = Formatter.new(opts.delete(:formatter))
     device = Device.new(opts)
 
     ::Logger.new(device).tap do |logger|
       logger.instance_variable_set(:@device, device)
       logger.extend(self)
       logger.extend(TaggedLogging)
-      logger.formatter = Formatter.new
+      logger.formatter = formatter
     end
   end
 

data/lib/logstash-logger/version.rb

@@ -1,3 +1,3 @@
 module LogStashLogger
-  VERSION = "0.11.0"
+  VERSION = "0.12.0"
 end

data/spec/formatter/base_spec.rb

@@ -0,0 +1,72 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Formatter::Base do
+  include_context "formatter"
+
+  describe "#build_event" do
+    let(:event) { formatted_message }
+
+    describe "message type" do
+      context "string" do
+        it "puts the message into the message field" do
+          expect(event['message']).to eq(message)
+        end
+      end
+
+      context "JSON string" do
+        let(:message) do
+          { message: 'test', foo: 'bar' }.to_json
+        end
+
+        it "parses the JSON and merges into the event" do
+          expect(event['message']).to eq('test')
+          expect(event['foo']).to eq('bar')
+        end
+      end
+
+      context "hash" do
+        let(:message) do
+          { 'message' => 'test', 'foo' => 'bar' }
+        end
+
+        it "merges into the event" do
+          expect(event['message']).to eq('test')
+          expect(event['foo']).to eq('bar')
+        end
+      end
+
+      context "LogStash::Event" do
+        let(:message) { LogStash::Event.new("message" => "foo") }
+
+        it "returns a clone of the original event" do
+          expect(event['message']).to eq("foo")
+          expect(event).to_not equal(message)
+        end
+      end
+
+      context "fallback" do
+        let(:message) { [1, 2, 3] }
+
+        it "calls inspect" do
+          expect(event['message']).to eq(message.inspect)
+        end
+      end
+    end
+
+    describe "extra fields on the event" do
+      it "adds severity" do
+        expect(event['severity']).to eq(severity)
+      end
+
+      it "adds host" do
+        expect(event['host']).to eq(hostname)
+      end
+    end
+
+    describe "timestamp" do
+      it "ensures time is in ISO8601 format" do
+        expect(event.timestamp).to eq(time.iso8601(3))
+      end
+    end
+  end
+end

data/spec/formatter/cee_spec.rb

@@ -0,0 +1,15 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Formatter::Cee do
+  include_context "formatter"
+
+  it "outputs in CEE format" do
+    expect(formatted_message).to match(/\A@cee:/)
+  end
+
+  it "serializes the LogStash::Event data as JSON" do
+    json_data = formatted_message[/\A@cee:\s?(.*)\z/, 1]
+    json_message = JSON.parse(json_data)
+    expect(json_message["message"]).to eq(message)
+  end
+end

data/spec/formatter/cee_syslog_spec.rb

@@ -0,0 +1,43 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Formatter::CeeSyslog do
+  include_context "formatter"
+
+  describe "#call" do
+    let(:facility) { "facility" }
+
+    before do
+      allow(subject).to receive(:facility).and_return(facility)
+    end
+
+    it "outputs a facility before the @cee" do
+      expect(formatted_message).to match(/\A#{facility}:@cee:/)
+    end
+
+    it "serializes the LogStash::Event data as JSON" do
+      json_data = formatted_message[/\A#{facility}:@cee:\s?(.*)\Z/, 1]
+      json_message = JSON.parse(json_data)
+      expect(json_message["message"]).to eq(message)
+    end
+  end
+
+  describe "#facility" do
+    let(:host) { Socket.gethostname }
+
+    before do
+      formatted_message
+    end
+
+    it "includes hostname and progname" do
+      expect(subject.send(:facility)).to match(/\A#{host}\s#{progname}\z/)
+    end
+
+    context "without progname" do
+      let(:progname) { nil }
+
+      it "only includes hostname" do
+        expect(subject.send(:facility)).to match(/\A#{host}\z/)
+      end
+    end
+  end
+end

data/spec/formatter/json_lines_spec.rb

@@ -0,0 +1,14 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Formatter::JsonLines do
+  include_context "formatter"
+
+  it "outputs in JSON format" do
+    json_message = JSON.parse(formatted_message)
+    expect(json_message["message"]).to eq(message)
+  end
+
+  it "terminates with a line break" do
+    expect(formatted_message[-1]).to eq("\n")
+  end
+end

data/spec/formatter/json_spec.rb

@@ -0,0 +1,10 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Formatter::Json do
+  include_context "formatter"
+
+  it "outputs in JSON format" do
+    json_message = JSON.parse(formatted_message)
+    expect(json_message["message"]).to eq(message)
+  end
+end

data/spec/formatter/logstash_event_spec.rb

@@ -0,0 +1,10 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Formatter::LogStashEvent do
+  include_context "formatter"
+
+  it "outputs a LogStash::Event" do
+    expect(formatted_message).to be_a LogStash::Event
+    expect(formatted_message["message"]).to eq(message)
+  end
+end

data/spec/formatter_spec.rb

@@ -0,0 +1,52 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Formatter do
+  describe "#new" do
+    context "built in formatters" do
+      it "returns a new JsonLines formatter" do
+        expect(described_class.new(:json_lines)).to be_a LogStashLogger::Formatter::JsonLines
+      end
+
+      it "returns a new Json formatter" do
+        expect(described_class.new(:json)).to be_a LogStashLogger::Formatter::Json
+      end
+
+      it "returns a new Cee formatter" do
+        expect(described_class.new(:cee)).to be_a LogStashLogger::Formatter::Cee
+      end
+
+      it "returns a new CeeSyslog formatter" do
+        expect(described_class.new(:cee_syslog)).to be_a LogStashLogger::Formatter::CeeSyslog
+      end
+
+      it "returns a new LogStashEvent formatter" do
+        expect(described_class.new(:logstash_event)).to be_a LogStashLogger::Formatter::LogStashEvent
+      end
+    end
+
+    context "custom formatter" do
+      it "returns a new instance of a custom formatter class" do
+        expect(described_class.new(::Logger::Formatter)).to be_a ::Logger::Formatter
+      end
+
+      it "returns the same formatter instance if a custom formatter instance is passed in" do
+        formatter = ::Logger::Formatter.new
+        expect(described_class.new(formatter)).to eql(formatter)
+      end
+
+      it "returns a formatter proc if it is passed in" do
+        formatter = proc do |severity, time, progname, msg|
+          msg
+        end
+        expect(described_class.new(formatter)).to eql(formatter)
+      end
+
+      it "returns a formatter lambda if it is passed in" do
+        formatter = lambda do |severity, time, progname, msg|
+          msg
+        end
+        expect(described_class.new(formatter)).to eql(formatter)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -75,3 +75,14 @@ RSpec.shared_context 'device' do
   let(:stdout_uri_config)  { {uri: stdout_uri} }
   let(:stderr_uri_config)  { {uri: stderr_uri} }
 end
+
+RSpec.shared_context 'formatter' do
+  let(:severity) { "DEBUG" }
+  let(:time) { Time.now }
+  let(:progname) { "ruby" }
+  let(:message) { "foo" }
+  let(:hostname) { Socket.gethostname }
+  let(:formatted_message) do
+    subject.call(severity, time, progname, message)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-logger
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - David Butler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-15 00:00:00.000000000 Z
+date: 2015-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-event
@@ -188,6 +188,12 @@ files:
 - lib/logstash-logger/device/udp.rb
 - lib/logstash-logger/device/unix.rb
 - lib/logstash-logger/formatter.rb
+- lib/logstash-logger/formatter/base.rb
+- lib/logstash-logger/formatter/cee.rb
+- lib/logstash-logger/formatter/cee_syslog.rb
+- lib/logstash-logger/formatter/json.rb
+- lib/logstash-logger/formatter/json_lines.rb
+- lib/logstash-logger/formatter/logstash_event.rb
 - lib/logstash-logger/logger.rb
 - lib/logstash-logger/railtie.rb
 - lib/logstash-logger/tagged_logging.rb
@@ -215,6 +221,13 @@ files:
 - spec/device/udp_spec.rb
 - spec/device/unix_spec.rb
 - spec/device_spec.rb
+- spec/formatter/base_spec.rb
+- spec/formatter/cee_spec.rb
+- spec/formatter/cee_syslog_spec.rb
+- spec/formatter/json_lines_spec.rb
+- spec/formatter/json_spec.rb
+- spec/formatter/logstash_event_spec.rb
+- spec/formatter_spec.rb
 - spec/logger_spec.rb
 - spec/rails_spec.rb
 - spec/spec_helper.rb
@@ -258,6 +271,13 @@ test_files:
 - spec/device/udp_spec.rb
 - spec/device/unix_spec.rb
 - spec/device_spec.rb
+- spec/formatter/base_spec.rb
+- spec/formatter/cee_spec.rb
+- spec/formatter/cee_syslog_spec.rb
+- spec/formatter/json_lines_spec.rb
+- spec/formatter/json_spec.rb
+- spec/formatter/logstash_event_spec.rb
+- spec/formatter_spec.rb
 - spec/logger_spec.rb
 - spec/rails_spec.rb
 - spec/spec_helper.rb