logstash-logger 0.26.1 → 1.1.0

data/spec/device/http_spec.rb

@@ -0,0 +1,176 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Device::HTTP do
+  include_context 'device'
+
+  it "Post event to HTTP" do
+    expect(Net::HTTP).to receive(:post).and_return(Net::HTTPOK.new('1.1', '200', 'OK'))
+    http_device.write('test')
+  end
+
+  describe "body format" do
+    let(:url) { 'http://localhost' }
+    let(:messages) { ["{\"a\":1}\n", "{\"b\":2}"] }
+
+    it "sends a JSON array by default" do
+      device = LogStashLogger::Device.new(type: :http, url: url, sync: true)
+      expect(Net::HTTP).to receive(:post)
+        .with(URI(url), "[{\"a\":1}\n,{\"b\":2}]", anything)
+        .and_return(Net::HTTPOK.new('1.1', '200', 'OK'))
+      device.write_batch(messages)
+    end
+
+    it "sends a JSON array with format: :json_array" do
+      device = LogStashLogger::Device.new(type: :http, url: url, sync: true, format: :json_array)
+      expect(Net::HTTP).to receive(:post)
+        .with(URI(url), "[{\"a\":1}\n,{\"b\":2}]", anything)
+        .and_return(Net::HTTPOK.new('1.1', '200', 'OK'))
+      device.write_batch(messages)
+    end
+
+    it "sends newline-delimited JSON with format: :ndjson" do
+      device = LogStashLogger::Device.new(type: :http, url: url, sync: true, format: :ndjson)
+      expect(Net::HTTP).to receive(:post)
+        .with(URI(url), "{\"a\":1}\n{\"b\":2}\n", anything)
+        .and_return(Net::HTTPOK.new('1.1', '200', 'OK'))
+      device.write_batch(messages)
+    end
+
+    it "accepts the format as a string" do
+      device = LogStashLogger::Device.new(type: :http, url: url, sync: true, format: "ndjson")
+      expect(Net::HTTP).to receive(:post)
+        .with(URI(url), "{\"a\":1}\n{\"b\":2}\n", anything)
+        .and_return(Net::HTTPOK.new('1.1', '200', 'OK'))
+      device.write_batch(messages)
+    end
+
+    it "raises an error for an unknown format" do
+      expect {
+        LogStashLogger::Device.new(type: :http, url: url, sync: true, format: :xml)
+      }.to raise_error(ArgumentError, /Unknown HTTP body format: :xml/)
+    end
+  end
+
+  describe "content type" do
+    let(:url) { 'http://localhost' }
+
+    def expect_content_type(type)
+      expect(Net::HTTP).to receive(:post)
+        .with(anything, anything, {"Content-Type" => type})
+        .and_return(Net::HTTPOK.new('1.1', '200', 'OK'))
+    end
+
+    it "is application/json by default" do
+      device = LogStashLogger::Device.new(type: :http, url: url, sync: true)
+      expect_content_type("application/json")
+      device.write_batch(["{}"])
+    end
+
+    it "is application/x-ndjson for format: :ndjson" do
+      device = LogStashLogger::Device.new(type: :http, url: url, sync: true, format: :ndjson)
+      expect_content_type("application/x-ndjson")
+      device.write_batch(["{}"])
+    end
+
+    it "can be overridden with content_type" do
+      device = LogStashLogger::Device.new(type: :http, url: url, sync: true, format: :ndjson, content_type: "application/json")
+      expect_content_type("application/json")
+      device.write_batch(["{}"])
+    end
+  end
+
+  it "does not raise on a successful response" do
+    allow(Net::HTTP).to receive(:post).and_return(Net::HTTPOK.new('1.1', '200', 'OK'))
+    expect { http_device.write_batch(['test']) }.not_to raise_error
+  end
+
+  it "raises on an error response" do
+    allow(Net::HTTP).to receive(:post).and_return(Net::HTTPInternalServerError.new('1.1', '500', 'Internal Server Error'))
+    expect { http_device.write_batch(['test']) }.to raise_error(RuntimeError, "HTTP log upload failed: 500 Internal Server Error")
+  end
+
+  it "raises on a client error response" do
+    allow(Net::HTTP).to receive(:post).and_return(Net::HTTPUnauthorized.new('1.1', '401', 'Unauthorized'))
+    expect { http_device.write_batch(['test']) }.to raise_error(RuntimeError, /401 Unauthorized/)
+  end
+end
+
+describe LogStashLogger::Device::HTTP, 'records the collector rejects' do
+  let(:url) { 'http://localhost' }
+  let(:error_log) { StringIO.new }
+  let(:sent) { [] }
+  let(:small) { (1..5).map { |i| %({"n":#{i}}) } }
+
+  def device(**opts)
+    LogStashLogger::Device.new({ type: :http, url: url, sync: false, buffer_max_interval: 3600,
+                                 buffer_flush_at_exit: false, error_logger: ::Logger.new(error_log) }.merge(opts))
+  end
+
+  # Stubs the HTTP endpoint: any request containing a "BAD" record gets the
+  # given error status, and every other request succeeds and is recorded.
+  def stub_endpoint_rejecting_bad_records(status:, message:)
+    allow(Net::HTTP).to receive(:post) do |_url, body, _headers|
+      if body.include?('BAD')
+        klass = Net::HTTPResponse::CODE_TO_OBJ[status]
+        klass.new('1.1', status, message)
+      else
+        sent << body
+        Net::HTTPOK.new('1.1', '200', 'OK')
+      end
+    end
+  end
+
+  it 'delivers the other records and drops only the rejected one' do
+    stub_endpoint_rejecting_bad_records(status: '413', message: 'Payload Too Large')
+    device.write_batch(small + ['"BAD"'])
+    expect(sent).to eq(small.map { |m| "[#{m}]" })
+    expect(error_log.string).to include('dropping a record of 5 bytes that the collector rejected: HTTP 413')
+  end
+
+  it 'drops a rejected single-record batch without raising' do
+    stub_endpoint_rejecting_bad_records(status: '400', message: 'Bad Request')
+    expect { device.write_batch(['"BAD"']) }.not_to raise_error
+    expect(error_log.string).to include('HTTP 400')
+  end
+
+  it 'clears a buffered batch instead of retrying the rejected record forever' do
+    stub_endpoint_rejecting_bad_records(status: '422', message: 'Unprocessable Entity')
+    dev = device
+    small.each { |m| dev.write(m) }
+    dev.write('"BAD"')
+    dev.buffer_flush(force: true)
+    expect(dev.instance_variable_get(:@buffer_state)[:pending_count]).to eq(0)
+    expect(sent.size).to eq(small.size)
+  end
+
+  # Errors that do not depend on the records, so the batch is retried
+  {
+    '401' => 'Unauthorized',          # missing or wrong credentials
+    '403' => 'Forbidden',             # credentials not allowed
+    '404' => 'Not Found',             # wrong URL, or the collector is not deployed yet
+    '429' => 'Too Many Requests',     # rate limited
+    '500' => 'Internal Server Error', # collector error
+    '503' => 'Service Unavailable',   # collector down or restarting
+  }.each do |status, message|
+    it "retries the whole batch on HTTP #{status} #{message}" do
+      stub_endpoint_rejecting_bad_records(status: status, message: message)
+      expect { device.write_batch(['"BAD"', 'ok']) }.to raise_error(RuntimeError, /HTTP log upload failed: #{status}/)
+      expect(error_log.string).to include("HTTP log upload failed: #{status}")
+    end
+  end
+
+  it 'raises if a record sent on its own hits a retryable error' do
+    calls = 0
+    allow(Net::HTTP).to receive(:post) do
+      calls += 1
+      calls == 1 ? Net::HTTPRequestEntityTooLarge.new('1.1', '413', 'Too Large') : Net::HTTPServiceUnavailable.new('1.1', '503', 'Unavailable')
+    end
+    expect { device.write_batch(['a', 'b']) }.to raise_error(RuntimeError, /503/)
+  end
+
+  it 'sends records that are not Strings as text' do
+    allow(Net::HTTP).to receive(:post) { |_u, body, _h| sent << body; Net::HTTPOK.new('1.1', '200', 'OK') }
+    device(format: :ndjson).write_batch([123, '{"a":1}'])
+    expect(sent).to eq(["123\n{\"a\":1}\n"])
+  end
+end

data/spec/device/kafka_spec.rb

@@ -1,32 +1,447 @@
 require 'logstash-logger'
+# The device requires ruby-kafka only when the first one is built. Load it here
+# so the classes reopened below and Kafka::SslContext exist in any test order.
+require 'ruby-kafka'
+
+describe LogStashLogger::Device::Kafka::TLSConfiguration do
+  context "with TLS" do
+    let(:complete_bundle) do
+      # NOTE these keys are obviously fake. We don't actually make connections
+      {
+        ssl_ca_cert: "----BEGIN CERT---- lkajdslkajdsjk ----END CERT---",
+        ssl_client_cert:      "----BEGIN CERT---- lkajdslkajdsjk ----END CERT---",
+        ssl_client_cert_key: "----PRIVATE---- lkajdslkajdsjk ----PRIVATE---",
+      }
+    end
+
+    context "when complete params are passed in" do
+      let(:instance) { described_class.new(complete_bundle) }
+
+      it "returns an empty hash when no ssl params are initialized" do
+        expect(instance.cert_bundle).to_not be_empty
+        expect(instance.valid?).to be_truthy
+      end
+    end
+
+    context "when incomplete params are passed in" do
+      [:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key].each do |param|
+        it "fails with an ArgumentError when #{param} no provided" do
+          opts = complete_bundle
+          opts[param] = nil
+          instance = described_class.new(opts)
+          expect(instance.cert_bundle).to be_empty
+          expect(instance.valid?).to be_falsey
+        end
+      end
+    end
+  end
+
+  context "without TLS" do
+    let(:instance) { subject }
+
+    it "returns an empty hash when no ssl params are initialized" do
+      expect(instance.cert_bundle).to be_empty
+      expect(instance.valid?).to be_truthy
+    end
+  end
+
+  context "#invalid?" do
+    it 'is just the opposite of valid?' do
+      expect(subject).to receive(:valid?).and_return(false)
+      expect(subject.invalid?).to be_truthy
+
+      expect(subject).to receive(:valid?).and_return(true)
+      expect(subject.invalid?).to be_falsey
+    end
+  end
+end
 
 describe LogStashLogger::Device::Kafka do
   include_context 'device'
 
-  let(:producer) { double("Poseidon::Producer") }
+  let(:broker_hosts) { "localhost:9300 localhost:9232" }
+  let(:opts) do
+    { topic: 'hello-world',
+      brokers: broker_hosts,
+    }
+  end
+  let(:instance) { LogStashLogger::Device::Kafka.new(opts) }
+  let(:brokers) { %w(localhost:9300 localhost:9232) }
+
+  describe "initializing" do
+    context "uri-derived options" do
+      it "normalizes host/port/path into brokers and topic" do
+        instance = described_class.new(
+          host: 'localhost',
+          port: 9092,
+          path: '/logstash'
+        )
+
+        expect(instance.brokers).to eq(['localhost:9092'])
+        expect(instance.topic).to eq('logstash')
+      end
+    end
+
+    context "brokers" do
+      context "when array" do
+        it "sets the brokers array to @brokers" do
+          instance = LogStashLogger::Device::Kafka.new(opts.merge({brokers: brokers}))
+
+          expect(instance.brokers).to eql(brokers)
+        end
+
+        it 'sets the brokers to an array if a string is passed in' do
+          instance = LogStashLogger::Device::Kafka.new(opts.merge({brokers: broker_hosts}))
+          expect(instance.brokers).to eql(brokers)
+        end
+      end
+
+      context "when missing or empty" do
+        it "raises an error if brokers are nil" do
+          expect {
+            described_class.new(topic: 'hello-world', brokers: nil)
+          }.to raise_error(ArgumentError)
+        end
+
+        it "raises an error if brokers are an empty array" do
+          expect {
+            described_class.new(topic: 'hello-world', brokers: [])
+          }.to raise_error(ArgumentError)
+        end
+
+        it "raises an error if brokers are blank" do
+          expect {
+            described_class.new(topic: 'hello-world', brokers: "  ")
+          }.to raise_error(ArgumentError)
+        end
+      end
+    end
+
+    context "topic" do
+      it 'sets the topic to the option provided' do
+        instance = described_class.new(topic: 'hello-world', brokers: brokers)
+        expect(instance.topic).to eql('hello-world')
+      end
+
+      it 'raises an exception if no topic is set' do
+        expect {
+          described_class.new(topic: nil)
+        }.to raise_error(ArgumentError)
+      end
+
+      it 'raises an exception if the topic is blank' do
+        expect {
+          described_class.new(topic: "  ", brokers: brokers)
+        }.to raise_error(ArgumentError)
+      end
+    end
+
+    context "Client Introspection" do
+      # YUCK! ruby-kafka does not presently allow reading certain variables
+      module ::Kafka
+        class Client
+          attr_reader :connection_builder
+        end
+      end
+
+      module ::Kafka
+        class ConnectionBuilder
+          attr_reader :ssl_context, :client_id
+        end
+      end
+
+      context "client_id" do
+        it 'sets a client_id when connecting if one is passed in to the options' do
+          instance = described_class.new(opts.merge(client_id: 'hello-world'))
+          expect(instance.client_id).to eql('hello-world')
+          expect(instance.connection.connection_builder.client_id).to eql('hello-world')
+        end
+      end
+
+      context "cert bundle" do
+        let(:complete_bundle) do
+          # NOTE these keys are obviously fake. We don't actually make connections
+          {
+            ssl_ca_cert: "----BEGIN CERT---- lkajdslkajdsjk ----END CERT---",
+            ssl_client_cert:      "----BEGIN CERT---- lkajdslkajdsjk ----END CERT---",
+            ssl_client_cert_key: "----PRIVATE---- lkajdslkajdsjk ----PRIVATE---",
+          }
+        end
+
+        context "no certs" do
+          it 'creates a connection without an ssl_context' do
+            connection = instance.connection
+            expect(connection.connection_builder.ssl_context).to be_nil
+          end
+        end
+
+        context "partial certs passed in" do
+          it 'fails if the complete cert suite is not passed in' do
+            [:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key].each do |param|
+              base_opts = opts
+              invalid_opts = complete_bundle.merge(base_opts)
+              invalid_opts[param] = nil
+              expect {
+                LogStashLogger::Device::Kafka.new(invalid_opts).connection
+              }.to raise_error( ArgumentError )
+            end
+          end
+        end
+
+        context "complete cert bundle" do
+
+          it 'correctly passes in the cert bundle to the Kafka Client' do
+            certopts = complete_bundle.merge(opts)
+
+            ssl_context = double("ssl_context")
+            expect(::Kafka::SslContext).to receive(:build)
+              .with(hash_including(
+                ca_cert: certopts[:ssl_ca_cert],
+                client_cert: certopts[:ssl_client_cert],
+                client_cert_key: certopts[:ssl_client_cert_key],
+              ))
+              .and_return(ssl_context)
+
+            LogStashLogger::Device::Kafka.new(certopts).connection
+          end
+        end
+      end
+    end
+  end
+
+  describe "writing single message to broker" do
+    it 'writes the message to the topic' do
+      producer = double('producer', produce: true)
+      connect_double = double("connection", producer: producer)
+      instance = described_class.new(opts)
+
+      # NOTE: this is stubbing out the ruby-kafka API
+      expect(instance).to receive(:connection).and_return(connect_double)
+      expect(connect_double).to receive(:producer)
+      expect(producer).to receive(:produce).and_return(true)
+      expect(producer).to receive(:deliver_messages).and_return(true)
+      instance.write_one("hello world")
+    end
+
+    it 'is capabable of writing to a different topic than instantiated' do
+      producer = double('producer', produce: lambda {|_message, _topic| "hi" })
+      connect_double = double("connection", producer: producer)
+      instance = described_class.new(opts)
+
+      message = 'hello world'
+      topic   = 'my topic'
+      # NOTE: this is stubbing out the ruby-kafka API
+      expect(instance).to receive(:connection).and_return(connect_double)
+      expect(connect_double).to receive(:producer)
+      expect(producer).to receive(:produce).
+                            with(message, topic: topic).
+                            and_return(true)
+      expect(producer).to receive(:deliver_messages).and_return(true)
+
+      instance.write_one(message, topic)
+    end
+  end
+
+  describe "error handling" do
+    it "clears the producer and connection when an error occurs" do
+      producer = double('producer')
+      allow(producer).to receive(:produce).and_raise(StandardError, "boom")
+      allow(producer).to receive(:deliver_messages)
+
+      connection = double("connection", producer: producer, close: true)
+      instance = described_class.new(opts)
+
+      allow(instance).to receive(:connection).and_return(connection)
+      expect(connection).to receive(:close)
 
-  before(:each) do
-    allow(Poseidon::Producer).to receive(:new) { producer }
+      expect {
+        instance.write_one("hello world")
+      }.to raise_error(StandardError)
+
+      expect(instance.instance_variable_get(:@producer)).to be_nil
+      expect(instance.instance_variable_get(:@io)).to be_nil
+    end
   end
 
-  it "writes to a Kafka topic" do
-    expect(producer).to receive(:send_messages)
-    kafka_device.write "foo"
+  describe "buffer group" do
+    it "uses the topic so final flush does not pass a boolean topic" do
+      instance = described_class.new(opts)
+
+      expect(instance).to receive(:write_batch).with(['hello world'], 'hello-world')
+      instance.write('hello world')
+      instance.buffer_flush(final: true)
+    end
   end
 
-  it "defaults the Kafka hosts to ['localhost:9092']" do
-    expect(kafka_device.hosts).to eq(['localhost:9092'])
+  describe "writing a batch of messages to the broker" do
+    it "writes the messages to the topic" do
+      producer = double('producer', produce: lambda {|_message, _topic| "hi" })
+      connect_double = double("connection", producer: producer)
+      instance = described_class.new(opts)
+
+      messages = ['hello world', 'goodbye world']
+      topic   = 'my topic'
+      # NOTE: this is stubbing out the ruby-kafka API
+      expect(instance).to receive(:connection).and_return(connect_double)
+      expect(connect_double).to receive(:producer)
+      expect(producer).to receive(:produce).
+                            with(messages.first, topic: topic).
+                            and_return(true)
+      expect(producer).to receive(:produce).
+                            with(messages.last, topic: topic).
+                            and_return(true)
+      expect(producer).to receive(:deliver_messages).and_return(true)
+
+      instance.write_batch(messages, topic)
+    end
   end
 
-  it "defaults the Kafka topic to 'logstash'" do
-    expect(kafka_device.topic).to eq('logstash')
+  describe "connecting/reconnecting" do
+    it "sets @io when connecting" do
+      instance = described_class.new(opts)
+      connection = double("connection")
+
+      instance.instance_variable_set(:@io, connection)
+      expect(instance.connect).to eq(connection)
+      expect(instance.io).to eq(connection)
+    end
+
+    it "reconnects without raising" do
+      instance = described_class.new(opts)
+      connection = double("connection", close: true)
+
+      allow(instance).to receive(:connection).and_return(connection)
+      expect { instance.reconnect }.not_to raise_error
+    end
   end
 
-  it "defaults the Kafka producer to 'logstash-logger'" do
-    expect(kafka_device.producer).to eq('logstash-logger')
+  describe "producer lifecycle" do
+    it "memoizes the producer across writes" do
+      producer = double('producer', produce: true, deliver_messages: true)
+      connection = double("connection", producer: producer)
+      instance = described_class.new(opts)
+
+      expect(instance).to receive(:connection).and_return(connection)
+      expect(connection).to receive(:producer).once.and_return(producer)
+
+      instance.write_one("hello world")
+      instance.write_one("goodbye world")
+    end
+
+    it "shuts down the producer on close" do
+      instance = described_class.new(opts)
+      producer = double("producer", shutdown: true)
+      connection = double("connection", close: true)
+
+      instance.instance_variable_set(:@producer, producer)
+      instance.instance_variable_set(:@io, connection)
+
+      expect(producer).to receive(:shutdown)
+      expect(connection).to receive(:close)
+
+      instance.close
+    end
   end
 
-  it "defaults the Kafka backoff to 1" do
-    expect(kafka_device.backoff).to eq(1)
+  describe "closing connection" do
+    it "closes the Kafka connection when present" do
+      instance = described_class.new(opts)
+      connection = double("connection")
+      expect(connection).to receive(:close)
+
+      instance.instance_variable_set(:@io, connection)
+      instance.close
+
+      expect(instance.instance_variable_get(:@connection)).to be_nil
+    end
+  end
+end
+
+describe LogStashLogger::Device::Kafka, 'records the broker always rejects' do
+  # Stands in for a ruby-kafka producer: rejects any delivery that contains a
+  # record over the limit, the way a broker answers MESSAGE_TOO_LARGE.
+  class FakeKafkaProducer
+    LIMIT = 1_000
+
+    def initialize(delivered, errors = {})
+      @delivered = delivered
+      @errors = errors
+      @buffer = []
+    end
+
+    def produce(message, topic:)
+      raise ::Kafka::BufferOverflow, 'buffer full' if message.bytesize > LIMIT * 10
+      @buffer << message
+    end
+
+    def deliver_messages
+      if (error = @errors.delete(:next_delivery))
+        @buffer.clear
+        raise error
+      end
+      if @buffer.any? { |m| m.bytesize > LIMIT }
+        @buffer.clear
+        raise ::Kafka::MessageSizeTooLarge, 'message too large'
+      end
+      @delivered.concat(@buffer)
+      @buffer.clear
+    end
+
+    def shutdown; end
+  end
+
+  let(:delivered) { [] }
+  let(:errors) { {} }
+  let(:error_log) { StringIO.new }
+  let(:device) do
+    LogStashLogger::Device::Kafka.new(topic: 'logs', brokers: 'localhost:9092', sync: false,
+      buffer_max_interval: 3600, buffer_flush_at_exit: false, error_logger: ::Logger.new(error_log))
+  end
+
+  before do
+    allow(device).to receive(:connection) do
+      double('connection', producer: FakeKafkaProducer.new(delivered, errors), close: true)
+    end
+  end
+
+  let(:small) { (1..5).map { |i| %({"n":#{i}}) } }
+
+  it 'delivers the other records and drops only an oversized one' do
+    device.write_batch(small + ['x' * 2_000])
+    expect(delivered).to eq(small)
+    expect(error_log.string).to include('dropping a record of 2000 bytes that Kafka rejected: Kafka::MessageSizeTooLarge')
+  end
+
+  it 'drops a record the client buffer cannot hold' do
+    device.write_batch(small + ['x' * 20_000])
+    expect(delivered).to eq(small)
+    expect(error_log.string).to include('Kafka::BufferOverflow')
+  end
+
+  it 'delivers everything when the batch is only too large as a whole' do
+    allow(device).to receive(:connection) do
+      producer = FakeKafkaProducer.new(delivered, errors)
+      def producer.deliver_messages
+        raise ::Kafka::MessageSizeTooLarge, 'batch too large' if @buffer.size > 1
+        super
+      end
+      double('connection', producer: producer, close: true)
+    end
+    device.write_batch(small)
+    expect(delivered).to eq(small)
+  end
+
+  it 'still raises transient errors so the buffer retries the batch' do
+    errors[:next_delivery] = ::Kafka::ConnectionError.new('broker unavailable')
+    expect { device.write_batch(small) }.to raise_error(::Kafka::ConnectionError)
+  end
+
+  it 'clears the buffer instead of holding other records behind the rejected one' do
+    small.each { |m| device.write(m) }
+    device.write('x' * 2_000)
+    device.buffer_flush(force: true)
+
+    expect(delivered).to eq(small)
+    expect(device.instance_variable_get(:@buffer_state)[:pending_count]).to eq(0)
   end
 end

data/spec/device/kinesis_spec.rb

@@ -30,9 +30,12 @@ describe LogStashLogger::Device::Kinesis do
     response.records = [failed_record]
 
     expect(client).to receive(:put_records) { response }
-    expect(kinesis_device).to receive(:write).with("foo")
+    expect(kinesis_device).not_to receive(:write)
 
     kinesis_device.write_one "foo"
+    state = kinesis_device.instance_variable_get(:@buffer_state)
+    expect(state[:pending_items][nil]).to eq(["foo"])
+    expect(state[:pending_count]).to eq(1)
   end
 
   it "defaults the kinesis stream to logstash" do