logstash-logger 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 00421e5706d99d473e29844c80e2c50f471f2ca1
4
- data.tar.gz: a1ae4bd4936c8acecbc560391bda1e70f624ecb8
3
+ metadata.gz: 1b5f101aba35d73c9bb62e77c6a5be82321c54de
4
+ data.tar.gz: 85d7303ebb637bda83b42dec2794c7551983e22e
5
5
  SHA512:
6
- metadata.gz: 194c6aec69634dfd22112a120fa6c9dc5e459f82eae2e97f78b1e512907bb20d7f475edb7d60f66ffec6637a72cafd5ba9c1079ae6bf6bf83696aa2276570f00
7
- data.tar.gz: 495dfc7623df42e62418e20b274185bb2c1e40fa588a166fd09af97dc5878d4cf721aaaa463df7c6a93706a1aed8da49057ce00074fdf9b6cfb3b4a1a24fd67c
6
+ metadata.gz: 0ce2d236f92070efa47bf40cbc708a2fa3c12d65f1e611813b6170678ce4b00e5889661f0e9b9d3ce7760f0f6eef12ea1773eb9ac98b843f8ae57f0514d4463f
7
+ data.tar.gz: 5c038e89a27a7467a76da3a7715d2161ca5afdf3f1bd6ec25216c433aaedd979399cbc0d133dd96b20760970637543814ddea4f80314612df174d915f369f3d7
data/.travis.yml CHANGED
@@ -14,3 +14,4 @@ gemfile:
14
14
  matrix:
15
15
  allow_failures:
16
16
  - rbx-2
17
+ sudo: false
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ ## 0.12.0
2
+ - Support for other formatters, including custom formatters. [#56](https://github.com/dwbutler/logstash-logger/pull/56)
3
+ - Support for CEE output.
4
+
1
5
  ## 0.11.0
2
6
  - Support for balancer log device. [#55](https://github.com/dwbutler/logstash-logger/pull/55)
3
7
 
data/README.md CHANGED
@@ -28,7 +28,7 @@ Or install it yourself as:
28
28
 
29
29
  $ gem install logstash-logger
30
30
 
31
- ## Basic Usage
31
+ ## Usage Examples
32
32
 
33
33
  ```ruby
34
34
  require 'logstash-logger'
@@ -49,6 +49,12 @@ stdout_logger = LogStashLogger.new(type: :stdout)
49
49
  stderr_logger = LogStashLogger.new(type: :stderr)
50
50
  io_logger = LogStashLogger.new(type: :io, io: io)
51
51
 
52
+ # Use a different formatter
53
+ cee_logger = LogStashLogger.new(type: :tcp, host: 'logsene-receiver-syslog.sematext.com', port: 514, formatter: :cee_syslog)
54
+ custom_formatted_logger = LogStashLogger.new(type: :redis, formatter: MyCustomFormatter)
55
+ lambda_formatted_logger = LogStashLogger.new(type: :stdout, formatter: ->(severity, time, progname, msg) { "[#{progname}] #{msg}" })
56
+ ruby_default_formatter_logger = LogStashLogger.new(type: :file, path: 'log/development.log', formatter: ::Logger::Formatter)
57
+
52
58
  # Multiple Outputs
53
59
  multi_logger = LogStashLogger.new([{type: :file, path: 'log/development.log'}, {type: :udp, host: 'localhost', port: 5228}])
54
60
 
@@ -77,7 +83,7 @@ logger.tagged('foo') { logger.fatal('bar') }
77
83
  ## URI Configuration
78
84
  You can use a URI to configure your logstash logger instead of a hash. This is useful in environments
79
85
  such as Heroku where you may want to read configuration values from the environment. The URI scheme
80
- is `type://host:port/path`. Some sample URI configurations are given below.
86
+ is `type://host:port/path?key=value`. Some sample URI configurations are given below.
81
87
 
82
88
  ```
83
89
  udp://localhost:5228
@@ -97,7 +103,7 @@ Pass the URI into your logstash logger like so:
97
103
  logger = LogStashLogger.new(uri: ENV['LOGSTASH_URI'])
98
104
  ```
99
105
 
100
- ## Logstash Configuration
106
+ ## Logstash Listener Configuration
101
107
 
102
108
  In order for logstash to correctly receive and parse the events, you will need to
103
109
  configure and run a listener that uses the `json_lines` codec. For example, to receive
@@ -1,51 +1,48 @@
1
- require 'logger'
2
- require 'socket'
3
- require 'time'
1
+ require 'logstash-logger/formatter/base'
4
2
 
5
3
  module LogStashLogger
6
- HOST = ::Socket.gethostname
4
+ module Formatter
5
+ DEFAULT_FORMATTER = :json_lines
7
6
 
8
- class Formatter < ::Logger::Formatter
9
- include TaggedLogging::Formatter
7
+ autoload :LogStashEvent, 'logstash-logger/formatter/logstash_event'
8
+ autoload :Json, 'logstash-logger/formatter/json'
9
+ autoload :JsonLines, 'logstash-logger/formatter/json_lines'
10
+ autoload :Cee, 'logstash-logger/formatter/cee'
11
+ autoload :CeeSyslog, 'logstash-logger/formatter/cee_syslog'
10
12
 
11
- def call(severity, time, progname, message)
12
- event = build_event(message, severity, time)
13
- "#{event.to_json}\n"
13
+ def self.new(formatter_type)
14
+ build_formatter(formatter_type)
14
15
  end
15
16
 
16
- protected
17
+ def self.build_formatter(formatter_type)
18
+ formatter_type ||= DEFAULT_FORMATTER
17
19
 
18
- def build_event(message, severity, time)
19
- data = message
20
- if data.is_a?(String) && data.start_with?('{')
21
- data = (JSON.parse(message) rescue nil) || message
20
+ if custom_formatter_instance?(formatter_type)
21
+ formatter_type
22
+ elsif custom_formatter_class?(formatter_type)
23
+ formatter_type.new
24
+ else
25
+ formatter_klass(formatter_type).new
22
26
  end
27
+ end
23
28
 
24
- event = case data
25
- when LogStash::Event
26
- data.clone
27
- when Hash
28
- event_data = data.merge("@timestamp" => time)
29
- LogStash::Event.new(event_data)
30
- else
31
- LogStash::Event.new("message" => msg2str(data), "@timestamp" => time)
32
- end
33
-
34
- event['severity'] ||= severity
35
- #event.type = progname
36
-
37
- event['host'] ||= HOST
29
+ def self.formatter_klass(formatter_type)
30
+ case formatter_type.to_sym
31
+ when :json_lines then JsonLines
32
+ when :json then Json
33
+ when :logstash_event then LogStashEvent
34
+ when :cee then Cee
35
+ when :cee_syslog then CeeSyslog
36
+ else fail ArgumentError, 'Invalid formatter'
37
+ end
38
+ end
38
39
 
39
- current_tags.each { |tag| event.tag(tag) }
40
-
41
- LogStashLogger.configuration.customize_event_block.call(event) if LogStashLogger.configuration.customize_event_block.respond_to?(:call)
40
+ def self.custom_formatter_instance?(formatter_type)
41
+ formatter_type.respond_to?(:call)
42
+ end
42
43
 
43
- # In case Time#to_json has been overridden
44
- if event.timestamp.is_a?(Time)
45
- event.timestamp = event.timestamp.iso8601(3)
46
- end
47
-
48
- event
44
+ def self.custom_formatter_class?(formatter_type)
45
+ formatter_type.is_a?(Class) && formatter_type.method_defined?(:call)
49
46
  end
50
- end
47
+ end
51
48
  end
@@ -0,0 +1,52 @@
1
+ require 'logger'
2
+ require 'socket'
3
+ require 'time'
4
+
5
+ module LogStashLogger
6
+ module Formatter
7
+ HOST = ::Socket.gethostname
8
+
9
+ class Base < ::Logger::Formatter
10
+ include ::LogStashLogger::TaggedLogging::Formatter
11
+
12
+ def call(severity, time, progname, message)
13
+ @event = build_event(message, severity, time)
14
+ end
15
+
16
+ protected
17
+
18
+ def build_event(message, severity, time)
19
+ data = message
20
+ if data.is_a?(String) && data.start_with?('{'.freeze)
21
+ data = (JSON.parse(message) rescue nil) || message
22
+ end
23
+
24
+ event = case data
25
+ when LogStash::Event
26
+ data.clone
27
+ when Hash
28
+ event_data = data.merge("@timestamp".freeze => time)
29
+ LogStash::Event.new(event_data)
30
+ else
31
+ LogStash::Event.new("message".freeze => msg2str(data), "@timestamp".freeze => time)
32
+ end
33
+
34
+ event['severity'.freeze] ||= severity
35
+ #event.type = progname
36
+
37
+ event['host'.freeze] ||= HOST
38
+
39
+ current_tags.each { |tag| event.tag(tag) }
40
+
41
+ LogStashLogger.configuration.customize_event_block.call(event) if LogStashLogger.configuration.customize_event_block.respond_to?(:call)
42
+
43
+ # In case Time#to_json has been overridden
44
+ if event.timestamp.is_a?(Time)
45
+ event.timestamp = event.timestamp.iso8601(3)
46
+ end
47
+
48
+ event
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,10 @@
1
+ module LogStashLogger
2
+ module Formatter
3
+ class Cee < Base
4
+ def call(severity, time, progname, message)
5
+ super
6
+ "@cee:#{@event.to_json}"
7
+ end
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,20 @@
1
+ module LogStashLogger
2
+ module Formatter
3
+ class CeeSyslog < Cee
4
+ def call(severity, time, progname, message)
5
+ @cee = super
6
+ @progname = progname
7
+
8
+ "#{facility}:#{@cee}\n"
9
+ end
10
+
11
+ protected
12
+
13
+ def facility
14
+ @facility = "#{@event['host']}"
15
+ @facility << " #{@progname}" if @progname
16
+ @facility
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,10 @@
1
+ module LogStashLogger
2
+ module Formatter
3
+ class Json < Base
4
+ def call(severity, time, progname, message)
5
+ super
6
+ @event.to_json
7
+ end
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,10 @@
1
+ module LogStashLogger
2
+ module Formatter
3
+ class JsonLines < Base
4
+ def call(severity, time, progname, message)
5
+ super
6
+ "#{@event.to_json}\n"
7
+ end
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,9 @@
1
+ module LogStashLogger
2
+ module Formatter
3
+ class LogStashEvent < Base
4
+ def call(severity, time, progname, message)
5
+ super
6
+ end
7
+ end
8
+ end
9
+ end
@@ -4,13 +4,14 @@ require 'logstash-logger/tagged_logging'
4
4
  module LogStashLogger
5
5
  def self.new(*args)
6
6
  opts = extract_opts(*args)
7
+ formatter = Formatter.new(opts.delete(:formatter))
7
8
  device = Device.new(opts)
8
9
 
9
10
  ::Logger.new(device).tap do |logger|
10
11
  logger.instance_variable_set(:@device, device)
11
12
  logger.extend(self)
12
13
  logger.extend(TaggedLogging)
13
- logger.formatter = Formatter.new
14
+ logger.formatter = formatter
14
15
  end
15
16
  end
16
17
 
@@ -1,3 +1,3 @@
1
1
  module LogStashLogger
2
- VERSION = "0.11.0"
2
+ VERSION = "0.12.0"
3
3
  end
@@ -0,0 +1,72 @@
1
+ require 'logstash-logger'
2
+
3
+ describe LogStashLogger::Formatter::Base do
4
+ include_context "formatter"
5
+
6
+ describe "#build_event" do
7
+ let(:event) { formatted_message }
8
+
9
+ describe "message type" do
10
+ context "string" do
11
+ it "puts the message into the message field" do
12
+ expect(event['message']).to eq(message)
13
+ end
14
+ end
15
+
16
+ context "JSON string" do
17
+ let(:message) do
18
+ { message: 'test', foo: 'bar' }.to_json
19
+ end
20
+
21
+ it "parses the JSON and merges into the event" do
22
+ expect(event['message']).to eq('test')
23
+ expect(event['foo']).to eq('bar')
24
+ end
25
+ end
26
+
27
+ context "hash" do
28
+ let(:message) do
29
+ { 'message' => 'test', 'foo' => 'bar' }
30
+ end
31
+
32
+ it "merges into the event" do
33
+ expect(event['message']).to eq('test')
34
+ expect(event['foo']).to eq('bar')
35
+ end
36
+ end
37
+
38
+ context "LogStash::Event" do
39
+ let(:message) { LogStash::Event.new("message" => "foo") }
40
+
41
+ it "returns a clone of the original event" do
42
+ expect(event['message']).to eq("foo")
43
+ expect(event).to_not equal(message)
44
+ end
45
+ end
46
+
47
+ context "fallback" do
48
+ let(:message) { [1, 2, 3] }
49
+
50
+ it "calls inspect" do
51
+ expect(event['message']).to eq(message.inspect)
52
+ end
53
+ end
54
+ end
55
+
56
+ describe "extra fields on the event" do
57
+ it "adds severity" do
58
+ expect(event['severity']).to eq(severity)
59
+ end
60
+
61
+ it "adds host" do
62
+ expect(event['host']).to eq(hostname)
63
+ end
64
+ end
65
+
66
+ describe "timestamp" do
67
+ it "ensures time is in ISO8601 format" do
68
+ expect(event.timestamp).to eq(time.iso8601(3))
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,15 @@
1
+ require 'logstash-logger'
2
+
3
+ describe LogStashLogger::Formatter::Cee do
4
+ include_context "formatter"
5
+
6
+ it "outputs in CEE format" do
7
+ expect(formatted_message).to match(/\A@cee:/)
8
+ end
9
+
10
+ it "serializes the LogStash::Event data as JSON" do
11
+ json_data = formatted_message[/\A@cee:\s?(.*)\z/, 1]
12
+ json_message = JSON.parse(json_data)
13
+ expect(json_message["message"]).to eq(message)
14
+ end
15
+ end
@@ -0,0 +1,43 @@
1
+ require 'logstash-logger'
2
+
3
+ describe LogStashLogger::Formatter::CeeSyslog do
4
+ include_context "formatter"
5
+
6
+ describe "#call" do
7
+ let(:facility) { "facility" }
8
+
9
+ before do
10
+ allow(subject).to receive(:facility).and_return(facility)
11
+ end
12
+
13
+ it "outputs a facility before the @cee" do
14
+ expect(formatted_message).to match(/\A#{facility}:@cee:/)
15
+ end
16
+
17
+ it "serializes the LogStash::Event data as JSON" do
18
+ json_data = formatted_message[/\A#{facility}:@cee:\s?(.*)\Z/, 1]
19
+ json_message = JSON.parse(json_data)
20
+ expect(json_message["message"]).to eq(message)
21
+ end
22
+ end
23
+
24
+ describe "#facility" do
25
+ let(:host) { Socket.gethostname }
26
+
27
+ before do
28
+ formatted_message
29
+ end
30
+
31
+ it "includes hostname and progname" do
32
+ expect(subject.send(:facility)).to match(/\A#{host}\s#{progname}\z/)
33
+ end
34
+
35
+ context "without progname" do
36
+ let(:progname) { nil }
37
+
38
+ it "only includes hostname" do
39
+ expect(subject.send(:facility)).to match(/\A#{host}\z/)
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,14 @@
1
+ require 'logstash-logger'
2
+
3
+ describe LogStashLogger::Formatter::JsonLines do
4
+ include_context "formatter"
5
+
6
+ it "outputs in JSON format" do
7
+ json_message = JSON.parse(formatted_message)
8
+ expect(json_message["message"]).to eq(message)
9
+ end
10
+
11
+ it "terminates with a line break" do
12
+ expect(formatted_message[-1]).to eq("\n")
13
+ end
14
+ end
@@ -0,0 +1,10 @@
1
+ require 'logstash-logger'
2
+
3
+ describe LogStashLogger::Formatter::Json do
4
+ include_context "formatter"
5
+
6
+ it "outputs in JSON format" do
7
+ json_message = JSON.parse(formatted_message)
8
+ expect(json_message["message"]).to eq(message)
9
+ end
10
+ end
@@ -0,0 +1,10 @@
1
+ require 'logstash-logger'
2
+
3
+ describe LogStashLogger::Formatter::LogStashEvent do
4
+ include_context "formatter"
5
+
6
+ it "outputs a LogStash::Event" do
7
+ expect(formatted_message).to be_a LogStash::Event
8
+ expect(formatted_message["message"]).to eq(message)
9
+ end
10
+ end
@@ -0,0 +1,52 @@
1
+ require 'logstash-logger'
2
+
3
+ describe LogStashLogger::Formatter do
4
+ describe "#new" do
5
+ context "built in formatters" do
6
+ it "returns a new JsonLines formatter" do
7
+ expect(described_class.new(:json_lines)).to be_a LogStashLogger::Formatter::JsonLines
8
+ end
9
+
10
+ it "returns a new Json formatter" do
11
+ expect(described_class.new(:json)).to be_a LogStashLogger::Formatter::Json
12
+ end
13
+
14
+ it "returns a new Cee formatter" do
15
+ expect(described_class.new(:cee)).to be_a LogStashLogger::Formatter::Cee
16
+ end
17
+
18
+ it "returns a new CeeSyslog formatter" do
19
+ expect(described_class.new(:cee_syslog)).to be_a LogStashLogger::Formatter::CeeSyslog
20
+ end
21
+
22
+ it "returns a new LogStashEvent formatter" do
23
+ expect(described_class.new(:logstash_event)).to be_a LogStashLogger::Formatter::LogStashEvent
24
+ end
25
+ end
26
+
27
+ context "custom formatter" do
28
+ it "returns a new instance of a custom formatter class" do
29
+ expect(described_class.new(::Logger::Formatter)).to be_a ::Logger::Formatter
30
+ end
31
+
32
+ it "returns the same formatter instance if a custom formatter instance is passed in" do
33
+ formatter = ::Logger::Formatter.new
34
+ expect(described_class.new(formatter)).to eql(formatter)
35
+ end
36
+
37
+ it "returns a formatter proc if it is passed in" do
38
+ formatter = proc do |severity, time, progname, msg|
39
+ msg
40
+ end
41
+ expect(described_class.new(formatter)).to eql(formatter)
42
+ end
43
+
44
+ it "returns a formatter lambda if it is passed in" do
45
+ formatter = lambda do |severity, time, progname, msg|
46
+ msg
47
+ end
48
+ expect(described_class.new(formatter)).to eql(formatter)
49
+ end
50
+ end
51
+ end
52
+ end
data/spec/spec_helper.rb CHANGED
@@ -75,3 +75,14 @@ RSpec.shared_context 'device' do
75
75
  let(:stdout_uri_config) { {uri: stdout_uri} }
76
76
  let(:stderr_uri_config) { {uri: stderr_uri} }
77
77
  end
78
+
79
+ RSpec.shared_context 'formatter' do
80
+ let(:severity) { "DEBUG" }
81
+ let(:time) { Time.now }
82
+ let(:progname) { "ruby" }
83
+ let(:message) { "foo" }
84
+ let(:hostname) { Socket.gethostname }
85
+ let(:formatted_message) do
86
+ subject.call(severity, time, progname, message)
87
+ end
88
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-logger
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.11.0
4
+ version: 0.12.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - David Butler
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-08-15 00:00:00.000000000 Z
11
+ date: 2015-08-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: logstash-event
@@ -188,6 +188,12 @@ files:
188
188
  - lib/logstash-logger/device/udp.rb
189
189
  - lib/logstash-logger/device/unix.rb
190
190
  - lib/logstash-logger/formatter.rb
191
+ - lib/logstash-logger/formatter/base.rb
192
+ - lib/logstash-logger/formatter/cee.rb
193
+ - lib/logstash-logger/formatter/cee_syslog.rb
194
+ - lib/logstash-logger/formatter/json.rb
195
+ - lib/logstash-logger/formatter/json_lines.rb
196
+ - lib/logstash-logger/formatter/logstash_event.rb
191
197
  - lib/logstash-logger/logger.rb
192
198
  - lib/logstash-logger/railtie.rb
193
199
  - lib/logstash-logger/tagged_logging.rb
@@ -215,6 +221,13 @@ files:
215
221
  - spec/device/udp_spec.rb
216
222
  - spec/device/unix_spec.rb
217
223
  - spec/device_spec.rb
224
+ - spec/formatter/base_spec.rb
225
+ - spec/formatter/cee_spec.rb
226
+ - spec/formatter/cee_syslog_spec.rb
227
+ - spec/formatter/json_lines_spec.rb
228
+ - spec/formatter/json_spec.rb
229
+ - spec/formatter/logstash_event_spec.rb
230
+ - spec/formatter_spec.rb
218
231
  - spec/logger_spec.rb
219
232
  - spec/rails_spec.rb
220
233
  - spec/spec_helper.rb
@@ -258,6 +271,13 @@ test_files:
258
271
  - spec/device/udp_spec.rb
259
272
  - spec/device/unix_spec.rb
260
273
  - spec/device_spec.rb
274
+ - spec/formatter/base_spec.rb
275
+ - spec/formatter/cee_spec.rb
276
+ - spec/formatter/cee_syslog_spec.rb
277
+ - spec/formatter/json_lines_spec.rb
278
+ - spec/formatter/json_spec.rb
279
+ - spec/formatter/logstash_event_spec.rb
280
+ - spec/formatter_spec.rb
261
281
  - spec/logger_spec.rb
262
282
  - spec/rails_spec.rb
263
283
  - spec/spec_helper.rb