logstash-logger 0.26.1 → 1.0.0

data/spec/device/kafka_spec.rb

@@ -1,32 +1,355 @@
 require 'logstash-logger'
 
+describe LogStashLogger::Device::Kafka::TLSConfiguration do
+  context "with TLS" do
+    let(:complete_bundle) do
+      # NOTE these keys are obviously fake. We don't actually make connections
+      {
+        ssl_ca_cert: "----BEGIN CERT---- lkajdslkajdsjk ----END CERT---",
+        ssl_client_cert:      "----BEGIN CERT---- lkajdslkajdsjk ----END CERT---",
+        ssl_client_cert_key: "----PRIVATE---- lkajdslkajdsjk ----PRIVATE---",
+      }
+    end
+
+    context "when complete params are passed in" do
+      let(:instance) { described_class.new(complete_bundle) }
+
+      it "returns an empty hash when no ssl params are initialized" do
+        expect(instance.cert_bundle).to_not be_empty
+        expect(instance.valid?).to be_truthy
+      end
+    end
+
+    context "when incomplete params are passed in" do
+      [:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key].each do |param|
+        it "fails with an ArgumentError when #{param} no provided" do
+          opts = complete_bundle
+          opts[param] = nil
+          instance = described_class.new(opts)
+          expect(instance.cert_bundle).to be_empty
+          expect(instance.valid?).to be_falsey
+        end
+      end
+    end
+  end
+
+  context "without TLS" do
+    let(:instance) { subject }
+
+    it "returns an empty hash when no ssl params are initialized" do
+      expect(instance.cert_bundle).to be_empty
+      expect(instance.valid?).to be_truthy
+    end
+  end
+
+  context "#invalid?" do
+    it 'is just the opposite of valid?' do
+      expect(subject).to receive(:valid?).and_return(false)
+      expect(subject.invalid?).to be_truthy
+
+      expect(subject).to receive(:valid?).and_return(true)
+      expect(subject.invalid?).to be_falsey
+    end
+  end
+end
+
 describe LogStashLogger::Device::Kafka do
   include_context 'device'
 
-  let(:producer) { double("Poseidon::Producer") }
+  let(:broker_hosts) { "localhost:9300 localhost:9232" }
+  let(:opts) do
+    { topic: 'hello-world',
+      brokers: broker_hosts,
+    }
+  end
+  let(:instance) { LogStashLogger::Device::Kafka.new(opts) }
+  let(:brokers) { %w(localhost:9300 localhost:9232) }
+
+  describe "initializing" do
+    context "uri-derived options" do
+      it "normalizes host/port/path into brokers and topic" do
+        instance = described_class.new(
+          host: 'localhost',
+          port: 9092,
+          path: '/logstash'
+        )
+
+        expect(instance.brokers).to eq(['localhost:9092'])
+        expect(instance.topic).to eq('logstash')
+      end
+    end
+
+    context "brokers" do
+      context "when array" do
+        it "sets the brokers array to @brokers" do
+          instance = LogStashLogger::Device::Kafka.new(opts.merge({brokers: brokers}))
+
+          expect(instance.brokers).to eql(brokers)
+        end
+
+        it 'sets the brokers to an array if a string is passed in' do
+          instance = LogStashLogger::Device::Kafka.new(opts.merge({brokers: broker_hosts}))
+          expect(instance.brokers).to eql(brokers)
+        end
+      end
+
+      context "when missing or empty" do
+        it "raises an error if brokers are nil" do
+          expect {
+            described_class.new(topic: 'hello-world', brokers: nil)
+          }.to raise_error(ArgumentError)
+        end
+
+        it "raises an error if brokers are an empty array" do
+          expect {
+            described_class.new(topic: 'hello-world', brokers: [])
+          }.to raise_error(ArgumentError)
+        end
+
+        it "raises an error if brokers are blank" do
+          expect {
+            described_class.new(topic: 'hello-world', brokers: "  ")
+          }.to raise_error(ArgumentError)
+        end
+      end
+    end
+
+    context "topic" do
+      it 'sets the topic to the option provided' do
+        instance = described_class.new(topic: 'hello-world', brokers: brokers)
+        expect(instance.topic).to eql('hello-world')
+      end
+
+      it 'raises an exception if no topic is set' do
+        expect {
+          described_class.new(topic: nil)
+        }.to raise_error(ArgumentError)
+      end
+
+      it 'raises an exception if the topic is blank' do
+        expect {
+          described_class.new(topic: "  ", brokers: brokers)
+        }.to raise_error(ArgumentError)
+      end
+    end
+
+    context "Client Introspection" do
+      # YUCK! ruby-kafka does not presently allow reading certain variables
+      module ::Kafka
+        class Client
+          attr_reader :connection_builder
+        end
+      end
+
+      module ::Kafka
+        class ConnectionBuilder
+          attr_reader :ssl_context, :client_id
+        end
+      end
+
+      context "client_id" do
+        it 'sets a client_id when connecting if one is passed in to the options' do
+          instance = described_class.new(opts.merge(client_id: 'hello-world'))
+          expect(instance.client_id).to eql('hello-world')
+          expect(instance.connection.connection_builder.client_id).to eql('hello-world')
+        end
+      end
+
+      context "cert bundle" do
+        let(:complete_bundle) do
+          # NOTE these keys are obviously fake. We don't actually make connections
+          {
+            ssl_ca_cert: "----BEGIN CERT---- lkajdslkajdsjk ----END CERT---",
+            ssl_client_cert:      "----BEGIN CERT---- lkajdslkajdsjk ----END CERT---",
+            ssl_client_cert_key: "----PRIVATE---- lkajdslkajdsjk ----PRIVATE---",
+          }
+        end
+
+        context "no certs" do
+          it 'creates a connection without an ssl_context' do
+            connection = instance.connection
+            expect(connection.connection_builder.ssl_context).to be_nil
+          end
+        end
+
+        context "partial certs passed in" do
+          it 'fails if the complete cert suite is not passed in' do
+            [:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key].each do |param|
+              base_opts = opts
+              invalid_opts = complete_bundle.merge(base_opts)
+              invalid_opts[param] = nil
+              expect {
+                LogStashLogger::Device::Kafka.new(invalid_opts).connection
+              }.to raise_error( ArgumentError )
+            end
+          end
+        end
 
-  before(:each) do
-    allow(Poseidon::Producer).to receive(:new) { producer }
+        context "complete cert bundle" do
+
+          it 'correctly passes in the cert bundle to the Kafka Client' do
+            certopts = complete_bundle.merge(opts)
+
+            ssl_context = double("ssl_context")
+            expect(::Kafka::SslContext).to receive(:build)
+              .with(hash_including(
+                ca_cert: certopts[:ssl_ca_cert],
+                client_cert: certopts[:ssl_client_cert],
+                client_cert_key: certopts[:ssl_client_cert_key],
+              ))
+              .and_return(ssl_context)
+
+            LogStashLogger::Device::Kafka.new(certopts).connection
+          end
+        end
+      end
+    end
   end
 
-  it "writes to a Kafka topic" do
-    expect(producer).to receive(:send_messages)
-    kafka_device.write "foo"
+  describe "writing single message to broker" do
+    it 'writes the message to the topic' do
+      producer = double('producer', produce: true)
+      connect_double = double("connection", producer: producer)
+      instance = described_class.new(opts)
+
+      # NOTE: this is stubbing out the ruby-kafka API
+      expect(instance).to receive(:connection).and_return(connect_double)
+      expect(connect_double).to receive(:producer)
+      expect(producer).to receive(:produce).and_return(true)
+      expect(producer).to receive(:deliver_messages).and_return(true)
+      instance.write_one("hello world")
+    end
+
+    it 'is capabable of writing to a different topic than instantiated' do
+      producer = double('producer', produce: lambda {|_message, _topic| "hi" })
+      connect_double = double("connection", producer: producer)
+      instance = described_class.new(opts)
+
+      message = 'hello world'
+      topic   = 'my topic'
+      # NOTE: this is stubbing out the ruby-kafka API
+      expect(instance).to receive(:connection).and_return(connect_double)
+      expect(connect_double).to receive(:producer)
+      expect(producer).to receive(:produce).
+                            with(message, topic: topic).
+                            and_return(true)
+      expect(producer).to receive(:deliver_messages).and_return(true)
+
+      instance.write_one(message, topic)
+    end
   end
 
-  it "defaults the Kafka hosts to ['localhost:9092']" do
-    expect(kafka_device.hosts).to eq(['localhost:9092'])
+  describe "error handling" do
+    it "clears the producer and connection when an error occurs" do
+      producer = double('producer')
+      allow(producer).to receive(:produce).and_raise(StandardError, "boom")
+      allow(producer).to receive(:deliver_messages)
+
+      connection = double("connection", producer: producer, close: true)
+      instance = described_class.new(opts)
+
+      allow(instance).to receive(:connection).and_return(connection)
+      expect(connection).to receive(:close)
+
+      expect {
+        instance.write_one("hello world")
+      }.to raise_error(StandardError)
+
+      expect(instance.instance_variable_get(:@producer)).to be_nil
+      expect(instance.instance_variable_get(:@io)).to be_nil
+    end
   end
 
-  it "defaults the Kafka topic to 'logstash'" do
-    expect(kafka_device.topic).to eq('logstash')
+  describe "buffer group" do
+    it "uses the topic so final flush does not pass a boolean topic" do
+      instance = described_class.new(opts)
+
+      expect(instance).to receive(:write_batch).with(['hello world'], 'hello-world')
+      instance.write('hello world')
+      instance.buffer_flush(final: true)
+    end
   end
 
-  it "defaults the Kafka producer to 'logstash-logger'" do
-    expect(kafka_device.producer).to eq('logstash-logger')
+  describe "writing a batch of messages to the broker" do
+    it "writes the messages to the topic" do
+      producer = double('producer', produce: lambda {|_message, _topic| "hi" })
+      connect_double = double("connection", producer: producer)
+      instance = described_class.new(opts)
+
+      messages = ['hello world', 'goodbye world']
+      topic   = 'my topic'
+      # NOTE: this is stubbing out the ruby-kafka API
+      expect(instance).to receive(:connection).and_return(connect_double)
+      expect(connect_double).to receive(:producer)
+      expect(producer).to receive(:produce).
+                            with(messages.first, topic: topic).
+                            and_return(true)
+      expect(producer).to receive(:produce).
+                            with(messages.last, topic: topic).
+                            and_return(true)
+      expect(producer).to receive(:deliver_messages).and_return(true)
+
+      instance.write_batch(messages, topic)
+    end
   end
 
-  it "defaults the Kafka backoff to 1" do
-    expect(kafka_device.backoff).to eq(1)
+  describe "connecting/reconnecting" do
+    it "sets @io when connecting" do
+      instance = described_class.new(opts)
+      connection = double("connection")
+
+      instance.instance_variable_set(:@io, connection)
+      expect(instance.connect).to eq(connection)
+      expect(instance.io).to eq(connection)
+    end
+
+    it "reconnects without raising" do
+      instance = described_class.new(opts)
+      connection = double("connection", close: true)
+
+      allow(instance).to receive(:connection).and_return(connection)
+      expect { instance.reconnect }.not_to raise_error
+    end
+  end
+
+  describe "producer lifecycle" do
+    it "memoizes the producer across writes" do
+      producer = double('producer', produce: true, deliver_messages: true)
+      connection = double("connection", producer: producer)
+      instance = described_class.new(opts)
+
+      expect(instance).to receive(:connection).and_return(connection)
+      expect(connection).to receive(:producer).once.and_return(producer)
+
+      instance.write_one("hello world")
+      instance.write_one("goodbye world")
+    end
+
+    it "shuts down the producer on close" do
+      instance = described_class.new(opts)
+      producer = double("producer", shutdown: true)
+      connection = double("connection", close: true)
+
+      instance.instance_variable_set(:@producer, producer)
+      instance.instance_variable_set(:@io, connection)
+
+      expect(producer).to receive(:shutdown)
+      expect(connection).to receive(:close)
+
+      instance.close
+    end
+  end
+
+  describe "closing connection" do
+    it "closes the Kafka connection when present" do
+      instance = described_class.new(opts)
+      connection = double("connection")
+      expect(connection).to receive(:close)
+
+      instance.instance_variable_set(:@io, connection)
+      instance.close
+
+      expect(instance.instance_variable_get(:@connection)).to be_nil
+    end
   end
 end

data/spec/device_spec.rb

@@ -38,6 +38,19 @@ describe LogStashLogger::Device do
       it { is_expected.to eq({type: 'udp', host: 'localhost', port: 5228, path: ''}) }
     end
 
+    context "when uri includes buffer_max_items" do
+      let(:uri_config) { {uri: "udp://localhost:5228?buffer_max_items=75"} }
+      it do
+        expect(parse_uri_config).to eq({
+          type: 'udp',
+          host: 'localhost',
+          port: 5228,
+          path: '',
+          buffer_max_items: 75,
+        })
+      end
+    end
+
     context "when uri is invalid" do
       let(:uri_config) { invalid_uri_config }
       specify { expect { parse_uri_config }.to raise_error(URI::InvalidURIError) }

data/spec/formatter/base_spec.rb

@@ -23,6 +23,51 @@ describe LogStashLogger::Formatter::Base do
         expect(subject.call(severity, time, progname, message)).to be_nil
       end
     end
+
+    context "when exception is raised" do
+      before do
+        allow(subject).to receive(:error_logger).and_return(Logger.new('/dev/null'))
+        allow(subject).to receive(:format_event).and_throw
+      end
+
+      it "logs an exception" do
+        expect(subject).to receive(:log_error)
+        subject.call(severity, time, progname, message)
+      end
+
+      it "returns a failed to format message" do
+        expect(subject.call(severity, time, progname, message)).to eq(LogStashLogger::Formatter::Base::FAILED_TO_FORMAT_MSG)
+      end
+    end
+  end
+
+  describe '#force_utf8_encoding' do
+    let(:message) { "foo x\xc3x".force_encoding('ASCII-8BIT').freeze }
+    let(:event) { LogStash::Event.new("message" => message) }
+
+    it 'returns the same event' do
+      expect(subject.send(:force_utf8_encoding, event)).to eq(event)
+    end
+
+    it 'forces the event message to UTF-8 encoding' do
+      subject.send(:force_utf8_encoding, event)
+      updated_event_data = event.instance_variable_get(:@data)
+      expect(updated_event_data['message'].encoding.name).to eq('UTF-8')
+    end
+
+    context 'frozen message string' do
+      let(:message) { "foo x\xc3x".dup.force_encoding('ASCII-8BIT').freeze }
+
+      it 'returns the same event' do
+        expect(subject.send(:force_utf8_encoding, event)).to eq(event)
+      end
+
+      it 'forces the event message to UTF-8 encoding' do
+        subject.send(:force_utf8_encoding, event)
+        updated_event_data = event.instance_variable_get(:@data)
+        expect(updated_event_data['message'].encoding.name).to eq('UTF-8')
+      end
+    end
   end
 
   describe "#build_event" do
@@ -81,7 +126,7 @@ describe LogStashLogger::Formatter::Base do
       end
 
       it "adds host" do
-        expect(event['host']).to eq(hostname)
+        expect(event['host']['hostname']).to eq(hostname)
       end
     end
 

data/spec/formatter/cee_syslog_spec.rb

@@ -22,21 +22,21 @@ describe LogStashLogger::Formatter::CeeSyslog do
   end
 
   describe "#build_facility" do
-    let(:host) { Socket.gethostname }
+    let(:host) { { 'hostname' => Socket.gethostname } }
 
     before do
       formatted_message
     end
 
     it "includes hostname and progname" do
-      expect(subject.send(:build_facility, host)).to match(/\A#{host}\s#{progname}\z/)
+      expect(subject.send(:build_facility, host)).to match(/\A#{host['hostname']}\s#{progname}\z/)
     end
 
     context "without progname" do
       let(:progname) { nil }
 
       it "only includes hostname" do
-        expect(subject.send(:build_facility, host)).to match(/\A#{host}\z/)
+        expect(subject.send(:build_facility, host)).to match(/\A#{host['hostname']}\z/)
       end
     end
   end

data/spec/formatter/json_lines_spec.rb

@@ -11,4 +11,27 @@ describe LogStashLogger::Formatter::JsonLines do
   it "terminates with a line break" do
     expect(formatted_message[-1]).to eq("\n")
   end
+
+  context 'encoding error is raised' do
+    let(:message) { "foo x\xc3x".force_encoding('ASCII-8BIT') }
+
+    before do
+      allow(subject).to receive(:error_logger).and_return(Logger.new('/dev/null'))
+    end
+
+    it 'logs the error' do
+      expect(subject).to receive(:log_error)
+      formatted_message
+    end
+
+    it 'forces the message encoding to utf8' do
+      expect(subject).to receive(:force_utf8_encoding)
+      formatted_message
+    end
+
+    it "outputs in JSON format with message encoding updated to utf8" do
+      json_message = JSON.parse(formatted_message)
+      expect(json_message["message"]).to eq(message.force_encoding(Encoding::UTF_8).scrub)
+    end
+  end
 end

data/spec/formatter/json_spec.rb

@@ -7,4 +7,53 @@ describe LogStashLogger::Formatter::Json do
     json_message = JSON.parse(formatted_message)
     expect(json_message["message"]).to eq(message)
   end
+
+  context 'encoding error is raised' do
+    let(:message) { "foo x\xc3x".force_encoding('ASCII-8BIT') }
+
+    before do
+      allow(subject).to receive(:error_logger).and_return(Logger.new('/dev/null'))
+    end
+
+    it 'logs the error' do
+      expect(subject).to receive(:log_error)
+      formatted_message
+    end
+
+    it 'forces the message encoding to utf8' do
+      expect(subject).to receive(:force_utf8_encoding)
+      formatted_message
+    end
+
+    it "outputs in JSON format with message encoding updated to utf8" do
+      json_message = JSON.parse(formatted_message)
+      expect(json_message["message"]).to eq(message.force_encoding(Encoding::UTF_8).scrub)
+    end
+  end
+
+  context "timestamp ordering" do
+    let(:message) { { message: 'test', foo: 'bar' } }
+
+    it "places @timestamp first in JSON output" do
+      timestamp_index = formatted_message.index('"@timestamp"')
+      message_index = formatted_message.index('"message"')
+
+      expect(timestamp_index).not_to be_nil
+      expect(message_index).not_to be_nil
+      expect(timestamp_index).to be < message_index
+    end
+  end
+
+  context "key ordering" do
+    let(:message) { { foo: 'bar', baz: 'qux' } }
+
+    it "preserves custom field order in JSON output" do
+      foo_index = formatted_message.index('"foo"')
+      baz_index = formatted_message.index('"baz"')
+
+      expect(foo_index).not_to be_nil
+      expect(baz_index).not_to be_nil
+      expect(foo_index).to be < baz_index
+    end
+  end
 end

data/spec/formatter_spec.rb

@@ -35,6 +35,23 @@ describe LogStashLogger::Formatter do
         end
       end
 
+      context "formatter class that accepts customize_event keyword arg in initializer" do
+        let(:formatter) do
+          Class.new do
+            def initialize(customize_event: nil)
+            end
+
+            def call
+            end
+          end
+        end
+
+        it "returns a new instance of the class" do
+          expect(formatter).to receive(:new).with(hash_including(:customize_event)).and_call_original
+          expect(subject).to be_a formatter
+        end
+      end
+
       context "formatter instance" do
         let(:formatter) { ::Logger::Formatter.new }
 
@@ -49,7 +66,7 @@ describe LogStashLogger::Formatter do
 
       context "formatter proc" do
         let(:formatter) do
-          proc { |severity, time, progname, msg| msg }
+          proc { |_severity, _time, _progname, msg| msg }
         end
 
         it "returns the same formatter proc" do
@@ -63,7 +80,7 @@ describe LogStashLogger::Formatter do
 
       context "formatter lambda" do
         let(:formatter) do
-          ->(severity, time, progname, msg) { msg }
+          ->(_severity, _time, _progname, msg) { msg }
         end
 
         it "returns the same formatter lambda" do

data/spec/logger_spec.rb

@@ -58,14 +58,14 @@ describe LogStashLogger do
 
     expect(listener_event['severity']).to eql('INFO')
     expect(listener_event['message']).to eq(message)
-    expect(listener_event['host']).to eq(hostname)
+    expect(listener_event['host']['hostname']).to eq(hostname)
   end
 
   it 'takes a logstash-formatted json string as input and writes out a logstash event' do
     logger.info(logstash_event.to_json)
 
     expect(listener_event['message']).to eq(logstash_event['message'])
-    expect(listener_event['host']).to eq(hostname)
+    expect(listener_event['host']['hostname']).to eq(hostname)
   end
 
   it 'takes a LogStash::Event as input and writes it out intact' do
@@ -74,7 +74,7 @@ describe LogStashLogger do
     expect(listener_event['message']).to eq(logstash_event['message'])
     expect(listener_event['severity']).to eq(logstash_event['severity'])
     expect(listener_event['@timestamp'].iso8601).to eq(logstash_event.timestamp.iso8601)
-    expect(listener_event['host']).to eq(hostname)
+    expect(listener_event['host']['hostname']).to eq(hostname)
   end
 
   it 'takes a data hash as input and writes out a logstash event' do
@@ -89,7 +89,7 @@ describe LogStashLogger do
     expect(listener_event['message']).to eq(data["message"])
     expect(listener_event['severity']).to eq(data['severity'])
     expect(listener_event['foo']).to eq(data['foo'])
-    expect(listener_event['host']).to eq(hostname)
+    expect(listener_event['host']['hostname']).to eq(hostname)
     expect(listener_event['@timestamp']).to_not be_nil
   end
 
@@ -99,7 +99,7 @@ describe LogStashLogger do
     logger.info(message)
 
     expect(listener_event['message']).to eq(message.inspect)
-    expect(listener_event['host']).to eq(hostname)
+    expect(listener_event['host']['hostname']).to eq(hostname)
     expect(listener_event['severity']).to eq('INFO')
   end
 

data/spec/multi_logger_spec.rb

@@ -56,4 +56,20 @@ describe LogStashLogger::MultiLogger do
       logger.info 'test'
     end
   end
+
+  it "can call .reset on each logger" do
+    subject.loggers.each do |logger|
+      expect(logger).to receive(:reset).and_call_original
+    end
+
+    subject.reset
+  end
+
+  it "delegates #log to loggers" do
+    subject.loggers.each do |logger|
+      expect(logger).to receive(:add).with(::Logger::DEBUG, "test", nil)
+    end
+
+    subject.log(::Logger::DEBUG, "test")
+  end
 end