logstash-integration-rabbitmq 0.0.1

data/spec/fixtures/testca/index.txt

@@ -0,0 +1,4 @@
+V	170510165759Z		01	unknown	/CN=localhost/O=server
+V	170510170001Z		02	unknown	/CN=localhost/O=client
+V	450505233839Z		05	unknown	/CN=localhost, O=client
+V	450505234253Z		06	unknown	/CN=localhost, O=server

data/spec/fixtures/testca/index.txt.attr

@@ -0,0 +1 @@
+unique_subject = yes

data/spec/fixtures/testca/index.txt.attr.old

@@ -0,0 +1 @@
+unique_subject = yes

data/spec/fixtures/testca/index.txt.old

@@ -0,0 +1,3 @@
+V	170510165759Z		01	unknown	/CN=localhost/O=server
+V	170510170001Z		02	unknown	/CN=localhost/O=client
+V	450505233839Z		05	unknown	/CN=localhost, O=client

data/spec/fixtures/testca/openssl.cnf

@@ -0,0 +1,53 @@
+[ ca ]
+default_ca = testca
+
+[ testca ]
+dir = .
+certificate = $dir/cacert.pem
+database = $dir/index.txt
+new_certs_dir = $dir/certs
+private_key = $dir/private/cakey.pem
+serial = $dir/serial
+
+default_crl_days = 7
+default_days = 10000
+default_md = sha256
+
+policy = testca_policy
+x509_extensions = certificate_extensions
+
+[ testca_policy ]
+commonName = supplied
+stateOrProvinceName = optional
+countryName = optional
+emailAddress = optional
+organizationName = optional
+organizationalUnitName = optional
+
+[ certificate_extensions ]
+basicConstraints = CA:false
+
+[ req ]
+default_bits = 2048
+default_keyfile = ./private/cakey.pem
+default_md = sha256
+prompt = yes
+distinguished_name = root_ca_distinguished_name
+x509_extensions = root_ca_extensions
+
+[ root_ca_distinguished_name ]
+commonName = hostname
+
+[ root_ca_extensions ]
+basicConstraints = CA:true
+keyUsage = keyCertSign, cRLSign
+
+[ client_ca_extensions ]
+basicConstraints = CA:false
+keyUsage = digitalSignature
+extendedKeyUsage = 1.3.6.1.5.5.7.3.2
+
+[ server_ca_extensions ]
+basicConstraints = CA:false
+keyUsage = keyEncipherment
+extendedKeyUsage = 1.3.6.1.5.5.7.3.1

data/spec/fixtures/testca/private/cakey.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAuP3x9a2NoWisFrddKseOJir4xvvLPP65bWUH5l8KkzvG5c+N
+PKfcxkzlHme9B3WWdGB3lx1buwyunbAejn9Uuwwc1QQA1kzBEGZgx7tgvmU0BRsg
+EL1pwd3v9Rc2cZxmNrQCqymGWNUqo4BfaNVlxoCHS23FckjwUJRfck54c3WqsunT
+jE6CoVtbYiKztp4fA38eJRyLgw1bZLOtNIySj1fRKywrCuxrGjdB7zd+c/ScsB8Y
+EFXwYEx2a9cL3qtnBrbIEh6V09E2cnIvsrdILzzk6OtNs9NnSA0XGcMMrBWHHs+i
+POdPX851inr3QFTnrPVtn4blGOGdZdR6QwNo2wIDAQABAoIBADoXZLflfChHmmK3
+ygX5DGZn8B9sSnIo+0mjBEwPZF6/0sGv34ZAoE+VLg3SPcXt4wVAlc1aZsfiP6M8
+/xt4WL80Gom57BlfmPDxdUrDSKoBVciUsAkRsfgzHXs1gt9CYcaj2IKvU4Tpy630
+TgG5oXAoRFQncG1nAjALp71ZbvGyZDOK2+/Vjs0NITyC+4srnafiF8pwLXmgm0Zj
+FVyCy8H/dwt7/pJKKkDX8Mq6kdvvROg8QDqTyCsTn3qAfDQUjO1BJje7x31E+lXW
+/FgUqs43ATU2HVczP0fbBZeGeFFTRycypmmSXYb9eZ05mDf8cm93V8KMPkTVh4Pu
+6/kF8AECgYEA8VpoTggar66f/0rfxX263q+L5YD8fwfuqcjLx/Bnuzpp1POhsTLD
+Gv0W5FcAf4i1jfnBevrk9q9WFnnUpUvX7Y+e1cvK9M5BuVRs77C4mE5IQ7hcHN6I
+vNgYihIK/PTVXYrT2j1EFLBwD2QWwO7BcV5aAiShlce7qC8fJeAhRtsCgYEAxDft
+QIUHm2MIzZD25q6fkki1Rx0ifOjO7AlK0NIyxQ+gsxZQrrUsSqcgC/W3XJSONeCY
+UjbHRQmhzkexA6ZFjgrL39qHL9x0kzle1dZ+oFGVQKiIMWn1WfEL0baDxMjFKDQt
+xF0SWRHZsIDPUjZzPTyfaPf8I148pIoSILB7BgECgYEA3679ioxiRz5dlNqM59ku
+DuK9klfoK8drPzoU+1nomZJ6sV2XFsZIIsQ7qiakFI7cTRgTZGoRODuqWqxRE11m
+Ywq/l8AHermKGjyPtdmgS7AJs5Gy9SKdsf/JRnWQb35uHQLkc5hid5ZKVUla+TaO
+XAao/uF6THnPhwEdKho+XQ8CgYEAiyQBiJQM/eIvVt4qRxCjNS975M7DKwJH4VcB
+h6zWtajMUtJLKmhs3Q9ACVsXyH3LjmcSfJI9ojYfWFC8NJNOlVgQlE+5N3ZD8DZp
+ioeMyZCwnuYjla7Gfh4RPIgJTpz0OfsuTSWWojSnQqNE4M6dz1nSzLO1RztHE4KZ
+MjcTNgECgYEAmeEA8KJzfsSgjjbhOmzxEJwRxzq0HTknTmWyEIamQBgFhaQ3AqZ+
+cZWkqS34foKxjLLXp3AMukpS50yRB98Wzaa7eoA/H0zXlRjA6id9qFuCXt+E9e/5
+sjPhQphj8473SsviSndT29JadTHgY8nSmXtxx+whO29D9Ods0laGb8A=
+-----END RSA PRIVATE KEY-----

data/spec/fixtures/testca/serial

@@ -0,0 +1 @@
+07

data/spec/fixtures/testca/serial.old

@@ -0,0 +1 @@
+06

data/spec/inputs/rabbitmq_spec.rb

@@ -0,0 +1,278 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/rabbitmq"
+require "thread"
+require 'logstash/event'
+
+Thread.abort_on_exception = true
+
+describe LogStash::Inputs::RabbitMQ do
+  let(:klass) { LogStash::Inputs::RabbitMQ }
+  let(:host) { "localhost" }
+  let(:port) { 5672 }
+  let(:exchange_type) { "topic" }
+  let(:exchange) { "myexchange" }
+  let(:queue) { "myqueue" }
+  let(:rabbitmq_settings) {
+    {
+      "host" => host,
+      "port" => port,
+      "queue" => queue,
+      "prefetch_count" => 123
+    }
+  }
+  let(:instance) { klass.new(rabbitmq_settings) }
+  let(:hare_info) { instance.instance_variable_get(:@hare_info) }
+
+  context "when connected" do
+    let(:connection) { double("MarchHare Connection") }
+    let(:channel) { double("Channel") }
+    let(:exchange) { double("Exchange") }
+    let(:channel) { double("Channel") }
+    let(:queue) { double("queue") }
+
+    # Doing this in a before block doesn't give us enough control over scope
+    before do
+      allow(instance).to receive(:connect!).and_call_original
+      allow(::MarchHare).to receive(:connect).and_return(connection)
+      allow(connection).to receive(:create_channel).and_return(channel)
+      allow(connection).to receive(:on_shutdown)
+      allow(connection).to receive(:on_blocked)
+      allow(connection).to receive(:on_unblocked)
+      allow(channel).to receive(:exchange).and_return(exchange)
+      allow(channel).to receive(:queue).and_return(queue)
+      allow(channel).to receive(:prefetch=)
+
+      allow(queue).to receive(:build_consumer).with(:on_cancellation => anything)
+      allow(queue).to receive(:subscribe_with).with(any_args)
+      allow(queue).to receive(:bind).with(any_args)
+    end
+
+    it "should default the codec to JSON" do
+      expect(instance.codec).to be_a(LogStash::Codecs::JSON)
+    end
+
+    describe "#connect!" do
+      subject { hare_info }
+
+      context "without an exchange declared" do
+        before do
+          instance.register
+          instance.setup!
+        end
+
+        it "should set the queue correctly" do
+          expect(subject.queue).to eql(queue)
+        end
+
+        it "should set the prefetch value correctly" do
+          expect(channel).to have_received(:prefetch=).with(123)
+        end
+      end
+
+      context "with an exchange declared" do
+        let(:exchange) { "exchange" }
+        let(:key) { "routing key" }
+        let(:rabbitmq_settings) { super.merge("exchange" => exchange, "key" => key, "exchange_type" => "fanout") }
+
+        before do
+          allow(instance).to receive(:declare_exchange!)
+        end
+
+        context "on run" do
+          before do
+            instance.register
+            instance.setup!
+          end
+
+          it "should bind to the exchange" do
+            expect(queue).to have_received(:bind).with(exchange, :routing_key => key)
+          end
+
+          it "should declare the exchange" do
+            expect(instance).to have_received(:declare_exchange!)
+          end
+        end
+
+        context "but not immediately available" do
+          before do
+            i = 0
+            allow(queue).to receive(:bind).with(any_args) do
+              i += 1
+              raise "foo" if i == 1
+            end
+          end
+
+          it "should reconnect" do
+            instance.register
+            instance.setup!
+            expect(queue).to have_received(:bind).with(any_args).twice()
+          end
+        end
+
+        context "initially unable to subscribe" do
+          before do
+            i = 0
+            allow(queue).to receive(:subscribe_with).with(any_args) do
+              i += 1
+              raise "sub error" if i == 1
+            end
+
+            it "should retry the subscribe" do
+              expect(queue).to have_receive(:subscribe_with).twice()
+            end
+          end
+        end
+      end
+    end
+  end
+
+  describe "with a live server", :integration => true do
+    let(:klass) { LogStash::Inputs::RabbitMQ }
+    let(:config) { {"host" => "127.0.0.1", "auto_delete" => true, "codec" => "plain", "add_field" => {"[@metadata][foo]" => "bar"} } }
+    let(:instance) { klass.new(config) }
+    let(:hare_info) { instance.instance_variable_get(:@hare_info) }
+    let(:output_queue) { Queue.new }
+
+    # Spawn a connection in the bg and wait up (n) seconds
+    def spawn_and_wait(instance)
+      instance.register
+
+      output_queue # materialize in this thread
+
+      Thread.new {
+        instance.run(output_queue)
+      }
+
+      20.times do
+        instance.connected? ? break : sleep(0.1)
+      end
+
+      # Extra time to make sure the consumer can attach
+      # Without this there's a chance the shutdown code will execute
+      # before consumption begins. This is tricky to do more elegantly
+      sleep 4
+    end
+
+    let(:test_connection) { MarchHare.connect(instance.send(:rabbitmq_settings)) }
+    let(:test_channel) { test_connection.create_channel }
+
+    before do
+      # Materialize the instance in the current thread to prevent dupes
+      # If you use multiple threads with lazy evaluation weird stuff happens
+      instance
+      spawn_and_wait(instance)
+
+      test_channel # Start up the test client as well
+    end
+
+    after do
+      instance.stop()
+      test_channel.close
+      test_connection.close
+    end
+
+    context "using defaults" do
+      it "should start, connect, and stop cleanly" do
+        expect(instance.connected?).to be_truthy
+      end
+    end
+
+    it "should have the correct prefetch value" do
+      expect(instance.instance_variable_get(:@hare_info).channel.prefetch).to eql(256)
+    end
+
+    describe "receiving a message with a queue + exchange specified" do
+      let(:config) { super.merge("queue" => queue_name, "exchange" => exchange_name, "exchange_type" => "fanout", "metadata_enabled" => true) }
+      let(:event) { output_queue.pop }
+      let(:exchange) { test_channel.exchange(exchange_name, :type => "fanout") }
+      let(:exchange_name) { "logstash-input-rabbitmq-#{rand(0xFFFFFFFF)}" }
+      #let(:queue) { test_channel.queue(queue_name, :auto_delete => true) }
+      let(:queue_name) { "logstash-input-rabbitmq-#{rand(0xFFFFFFFF)}" }
+
+      after do
+        exchange.delete
+      end
+
+      context "when the message has a payload but no message headers" do
+        before do
+          exchange.publish(message)
+        end
+
+        let(:message) { "Foo Message" }
+
+        it "should process the message and store the payload" do
+          expect(event.get("message")).to eql(message)
+        end
+
+        it "should save an empty message header hash" do
+          expect(event).to include("@metadata")
+          expect(event.get("@metadata")).to include("rabbitmq_headers")
+          expect(event.get("[@metadata][rabbitmq_headers]")).to eq({})
+        end
+      end
+
+      context "when message properties are available" do
+        before do
+          # Don't test every single property but select a few with
+          # different characteristics to get sufficient coverage.
+          exchange.publish("",
+                        :properties => {
+                          :app_id    => app_id,
+                          :timestamp => Java::JavaUtil::Date.new(epoch * 1000),
+                          :priority  => priority,
+                        })
+        end
+
+        let(:app_id) { "myapplication" }
+        # Randomize the epoch we test with but limit its range to signed
+        # ints to not assume all protocols and libraries involved use
+        # unsigned ints for epoch values.
+        let(:epoch) { rand(0x7FFFFFFF) }
+        let(:priority) { 5 }
+
+        it "should save message properties into a @metadata field" do
+          expect(event).to include("@metadata")
+          expect(event.get("@metadata")).to include("rabbitmq_properties")
+
+          props = event.get("[@metadata][rabbitmq_properties")
+          expect(props["app-id"]).to eq(app_id)
+          expect(props["delivery-mode"]).to eq(1)
+          expect(props["exchange"]).to eq(exchange_name)
+          expect(props["priority"]).to eq(priority)
+          expect(props["routing-key"]).to eq("")
+          expect(props["timestamp"]).to eq(epoch)
+        end
+      end
+
+      context "when message headers are available" do
+        before do
+          exchange.publish("", :properties => { :headers => headers })
+        end
+
+        let (:headers) {
+          {
+            "arrayvalue"  => [true, 123, "foo"],
+            "boolvalue"   => true,
+            "intvalue"    => 123,
+            "stringvalue" => "foo",
+          }
+        }
+
+        it "should save message headers into a @metadata field" do
+          expect(event).to include("@metadata")
+          expect(event.get("@metadata")).to include("rabbitmq_headers")
+          expect(event.get("[@metadata][rabbitmq_headers]")).to include(headers)
+        end
+
+        it "should properly decorate the event" do
+          expect(event.get("[@metadata][foo]")).to eq("bar")
+        end
+      end
+    end
+
+    describe LogStash::Inputs::RabbitMQ do
+      it_behaves_like "an interruptible input plugin"
+    end
+  end
+end

data/spec/outputs/rabbitmq_spec.rb

@@ -0,0 +1,228 @@
+# encoding: UTF-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/pipeline"
+require "logstash/outputs/rabbitmq"
+java_import java.util.concurrent.TimeoutException
+
+describe LogStash::Outputs::RabbitMQ do
+  let(:klass) { LogStash::Outputs::RabbitMQ }
+  let(:host) { "localhost" }
+  let(:port) { 5672 }
+  let(:exchange_type) { "topic" }
+  let(:exchange) { "myexchange" }
+  let(:key) { "mykey" }
+  let(:persistent) { true }
+  let(:rabbitmq_settings) {
+    {
+      "host" => host,
+      "port" => port,
+      "exchange_type" => exchange_type,
+      "exchange" => exchange,
+      "key" => key,
+      "persistent" => persistent
+    }
+  }
+  let(:instance) { klass.new(rabbitmq_settings) }
+  let(:hare_info) { instance.instance_variable_get(:@hare_info) }
+
+  shared_examples 'recovers from exception gracefully' do
+    it 'should execute publish twice due to a retry' do
+      expect(exchange).to have_received(:publish).twice
+    end
+
+    it 'should sleep for the retry' do
+      expect(instance).to have_received(:sleep_for_retry).once
+    end
+
+    it 'should send the correct message (twice)' do
+      expect(exchange).to have_received(:publish).with(encoded_event, anything).twice
+    end
+
+    it 'should send the correct metadata (twice)' do
+      expected_metadata = {:routing_key => event.sprintf(key), :properties => {:persistent => persistent }}
+      expect(exchange).to have_received(:publish).with(anything, expected_metadata).twice
+    end
+  end
+
+  it "should register as an output plugin" do
+    expect(LogStash::Plugin.lookup("output", "rabbitmq")).to eql(LogStash::Outputs::RabbitMQ)
+  end
+
+  context "when connected" do
+    let(:connection) { double("MarchHare Connection") }
+    let(:channel) { double("Channel") }
+    let(:exchange) { double("Exchange") }
+
+    before do
+      allow(instance).to receive(:connect!).and_call_original
+      allow(::MarchHare).to receive(:connect).and_return(connection)
+      allow(connection).to receive(:create_channel).and_return(channel)
+      allow(connection).to receive(:on_blocked)
+      allow(connection).to receive(:on_unblocked)
+      allow(connection).to receive(:on_shutdown)
+      allow(channel).to receive(:exchange).and_return(exchange)
+
+      instance.register
+    end
+
+    describe "#connect!" do
+      subject { hare_info }
+
+      it "should set the exchange correctly" do
+        expect(subject.exchange).to eql(exchange)
+      end
+    end
+
+    describe "#publish_encoded" do
+      let(:event) { LogStash::Event.new("foo" => "bar") }
+      let(:sprinted_key) { double("sprinted key") }
+      let(:encoded_event) { LogStash::Json.dump(event) }
+
+      describe "issuing the publish" do
+        before do
+          allow(exchange).to receive(:publish).with(any_args)
+          allow(event).to receive(:sprintf).with(key).and_return(sprinted_key)
+          instance.send(:publish, event, encoded_event)
+        end
+
+        it "should send the correct message" do
+          expect(exchange).to have_received(:publish).with(encoded_event, anything)
+        end
+
+        it "should send the correct metadata" do
+          expected_metadata = {:routing_key => sprinted_key, :properties => {:persistent => persistent }}
+
+          expect(exchange).to have_received(:publish).with(anything, expected_metadata)
+        end
+      end
+
+      context 'when an exception is encountered' do
+        let(:exception) { nil }
+
+        before do
+          i = 0
+          allow(instance).to receive(:connect!)
+          allow(instance).to receive(:sleep_for_retry)
+          allow(exchange).to receive(:publish).with(any_args) do
+            i += 1
+            raise exception if i == 1
+          end
+
+          instance.send(:publish, event, encoded_event)
+        end
+
+        context 'when it is a MarchHare exception' do
+          let(:exception) { MarchHare::Exception }
+          it_behaves_like 'recovers from exception gracefully'
+        end
+
+        context 'when it is a MarchHare::ChannelAlreadyClosed' do
+          let(:exception) { MarchHare::ChannelAlreadyClosed }
+          it_behaves_like 'recovers from exception gracefully'
+        end
+
+        context 'when it is a TimeoutException' do
+          let(:exception) { TimeoutException.new }
+          it_behaves_like 'recovers from exception gracefully'
+        end
+      end
+    end
+  end
+  describe "with a live server", :integration => true do
+    let(:klass) { LogStash::Outputs::RabbitMQ }
+    let(:exchange) { "myexchange" }
+    let(:exchange_type) { "topic" }
+    let(:priority) { 34 }
+    let(:default_plugin_config) {
+      {
+        "host" => "127.0.0.1",
+        "exchange" => exchange,
+        "exchange_type" => exchange_type,
+        "key" => "foo",
+        "message_properties" => {
+            "priority" => priority
+        }
+      }
+    }
+    let(:config) { default_plugin_config }
+    let(:instance) { klass.new(config) }
+    let(:hare_info) { instance.instance_variable_get(:@hare_info) }
+
+    # Spawn a connection in the bg and wait up (n) seconds
+    def spawn_and_wait(instance)
+      instance.register
+
+      20.times do
+        instance.connected? ? break : sleep(0.1)
+      end
+
+      # Extra time to make sure the output can attach
+      sleep 1
+    end
+    let(:message) { LogStash::Event.new("message" => "Foo Message", "extra_field" => "Blah") }
+    let(:encoded) { message.to_json }
+    let(:test_connection) { MarchHare.connect(instance.send(:rabbitmq_settings)) }
+    let(:test_channel) { test_connection.create_channel }
+    let(:test_queue) {
+      test_channel.queue("testq", :auto_delete => true).bind(exchange, :key => config["key"])
+    }
+
+    before do
+      # Materialize the instance in the current thread to prevent dupes
+      # If you use multiple threads with lazy evaluation weird stuff happens
+      instance
+      spawn_and_wait(instance)
+
+      test_channel # Start up the test client as well
+      test_queue
+    end
+
+    after do
+      instance.close()
+      test_channel.close
+      test_connection.close
+    end
+
+    context "using defaults" do
+      it "should start, connect, and stop cleanly" do
+        expect(instance.connected?).to be_truthy
+      end
+
+      it "should close cleanly" do
+        instance.close
+        expect(instance.connected?).to be_falsey
+      end
+
+      it 'applies per message settings' do
+        instance.multi_receive_encoded([[message, encoded]])
+        sleep 1.0
+
+        message, payload = test_queue.pop
+        expect(message.properties.to_s).to include("priority=#{priority}")
+      end
+    end
+
+    describe "sending a message with an exchange specified" do
+      let(:message) { LogStash::Event.new("message" => "Foo Message", "extra_field" => "Blah") }
+
+      before do
+        @received = nil
+        test_queue.subscribe do |metadata,payload|
+          @received = payload
+        end
+
+        instance.multi_receive_encoded([[message, encoded]])
+
+        until @received
+          sleep 1
+        end
+
+        @decoded = LogStash::Json.load(@received)
+      end
+
+      it "should process the message fully using the default (JSON) codec" do
+        expect(@decoded).to eql(LogStash::Json.load(LogStash::Json.dump(message)))
+      end
+    end
+  end
+end