logstash-output-elasticsearch 0.1.6 → 3.0.0

data/spec/unit/buffer_spec.rb

@@ -0,0 +1,118 @@
+require "logstash/outputs/elasticsearch"
+require "logstash/outputs/elasticsearch/buffer"
+
+describe LogStash::Outputs::ElasticSearch::Buffer do
+  class OperationTarget # Used to track buffer flushesn
+    attr_reader :buffer, :buffer_history, :receive_count
+    def initialize
+      @buffer = nil
+      @buffer_history = []
+      @receive_count = 0
+    end
+
+    def receive(buffer)
+      @receive_count += 1
+      @buffer_history << buffer.clone
+      @buffer = buffer
+    end
+  end
+
+  let(:logger) { Cabin::Channel.get }
+  let(:max_size) { 10 }
+  let(:flush_interval) { 2 }
+  # Used to track flush count
+  let(:operation_target) { OperationTarget.new() }
+  let(:operation) { proc {|buffer| operation_target.receive(buffer) } }
+  subject(:buffer){ LogStash::Outputs::ElasticSearch::Buffer.new(logger, max_size, flush_interval, &operation) }
+
+  after(:each) do
+    buffer.stop(do_flush=false)
+  end
+
+  it "should initialize cleanly" do
+    expect(buffer).to be_a(LogStash::Outputs::ElasticSearch::Buffer)
+  end
+
+  shared_examples("a buffer with two items inside") do
+    it "should add a pushed item to the buffer" do
+      buffer.synchronize do |data|
+        expect(data).to include(item1)
+        expect(data).to include(item2)
+      end
+    end
+
+    describe "interval flushing" do
+      before do
+        sleep flush_interval + 1
+      end
+
+      it "should flush the buffer after the interval has passed" do
+        expect(operation_target.receive_count).to eql(1)
+      end
+
+      it "should clear the buffer after a successful flush" do
+        expect(operation_target.buffer).to eql([])
+      end
+    end
+
+    describe "interval flushing a stopped buffer" do
+      before do
+        buffer.stop(do_flush=false)
+        sleep flush_interval + 1
+      end
+
+      it "should not flush if the buffer is stopped" do
+        expect(operation_target.receive_count).to eql(0)
+      end
+    end
+  end
+
+  describe "with a buffer push" do
+    let(:item1) { "foo" }
+    let(:item2) { "bar" }
+
+    describe "a buffer with two items pushed to it separately" do
+      before do
+        buffer << item1
+        buffer << item2
+      end
+
+      include_examples("a buffer with two items inside")
+    end
+
+    describe "a buffer with two items pushed to it in one operation" do
+      before do
+        buffer.push_multi([item1, item2])
+      end
+
+      include_examples("a buffer with two items inside")
+    end
+  end
+
+  describe "with an empty buffer" do
+    it "should not perform an operation if the buffer is empty" do
+      buffer.flush
+      expect(operation_target.receive_count).to eql(0)
+    end
+  end
+
+  describe "flushing with an operation that raises an error" do
+    class TestError < StandardError; end
+    let(:operation) { proc {|buffer| raise TestError, "A test" } }
+    let(:item) { double("item") }
+
+    before do
+      buffer << item
+    end
+
+    it "should raise an exception" do
+      expect { buffer.flush }.to raise_error(TestError)
+    end
+
+    it "should not clear the buffer" do
+      expect do
+        buffer.flush rescue TestError
+      end.not_to change(buffer, :contents)
+    end
+  end
+end

data/spec/unit/http_client_builder_spec.rb

@@ -0,0 +1,27 @@
+require "logstash/outputs/elasticsearch"
+require "logstash/outputs/elasticsearch/http_client"
+require "logstash/outputs/elasticsearch/http_client_builder"
+
+describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
+  describe "auth setup with url encodable passwords" do
+    let(:klass) { LogStash::Outputs::ElasticSearch::HttpClientBuilder }
+    let(:user) { "foo@bar"}
+    let(:password) {"baz@blah" }
+    let(:password_secured) do
+      secured = double("password")
+      allow(secured).to receive(:value).and_return(password)
+      secured
+    end
+    let(:options) { {"user" => user, "password" => password} }
+    let(:logger) { mock("logger") }
+    let(:auth_setup) { klass.setup_basic_auth(double("logger"), {"user" => user, "password" => password_secured}) }
+
+    it "should return the user verbatim" do
+      expect(auth_setup[:user]).to eql(user)
+    end
+
+    it "should return the password verbatim" do
+      expect(auth_setup[:password]).to eql(password)
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -0,0 +1,133 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch/http_client"
+require "java"
+
+describe LogStash::Outputs::ElasticSearch::HttpClient do
+  let(:base_options) { {:hosts => ["127.0.0.1"], :logger => Cabin::Channel.get }}
+
+  describe "Host/URL Parsing" do
+    subject { described_class.new(base_options) }
+
+    let(:true_hostname) { "my-dash.hostname" }
+    let(:ipv6_hostname) { "[::1]" }
+    let(:ipv4_hostname) { "127.0.0.1" }
+    let(:port) { 9202 }
+    let(:hostname_port) { "#{hostname}:#{port}"}
+    let(:http_hostname_port) { "http://#{hostname_port}"}
+    let(:https_hostname_port) { "https://#{hostname_port}"}
+    let(:http_hostname_port_path) { "http://#{hostname_port}/path"}
+
+    shared_examples("proper host handling") do
+      it "should properly transform a host:port string to a URL" do
+        expect(subject.send(:host_to_url, hostname_port)).to eql(http_hostname_port)
+      end
+
+      it "should raise an error when a partial URL is an invalid format" do
+        expect {
+          subject.send(:host_to_url, "#{hostname_port}/")
+        }.to raise_error(LogStash::ConfigurationError)
+      end
+
+      it "should not raise an error with a / for a path" do
+        expect(subject.send(:host_to_url, "#{http_hostname_port}/")).to eql("#{http_hostname_port}/")
+      end
+
+      it "should parse full URLs correctly" do
+        expect(subject.send(:host_to_url, http_hostname_port)).to eql(http_hostname_port)
+      end
+
+      it "should reject full URLs with usernames and passwords" do
+        expect {
+          subject.send(:host_to_url, "http://user:password@host.domain")
+        }.to raise_error(LogStash::ConfigurationError)
+      end
+
+      describe "ssl" do
+        it "should refuse to handle an http url when ssl is true" do
+          expect {
+            subject.send(:host_to_url, http_hostname_port, true)
+          }.to raise_error(LogStash::ConfigurationError)
+        end
+
+        it "should refuse to handle an https url when ssl is false" do
+          expect {
+            subject.send(:host_to_url, https_hostname_port, false)
+          }.to raise_error(LogStash::ConfigurationError)
+        end
+
+        it "should handle an ssl url correctly when SSL is nil" do
+          expect(subject.send(:host_to_url, https_hostname_port, nil)).to eql(https_hostname_port)
+        end
+
+        it "should raise an exception if an unexpected value is passed in" do
+          expect { subject.send(:host_to_url, https_hostname_port, {})}.to raise_error(ArgumentError)
+        end
+      end
+
+      describe "path" do
+        it "should allow paths in a url" do
+          expect(subject.send(:host_to_url, http_hostname_port_path, nil)).to eql(http_hostname_port_path)
+        end
+
+        it "should not allow paths in two places" do
+          expect {
+            subject.send(:host_to_url, http_hostname_port_path, false, "/otherpath")
+          }.to raise_error(LogStash::ConfigurationError)
+        end
+
+        it "should automatically insert a / in front of path overlays if needed" do
+          expect(subject.send(:host_to_url, http_hostname_port, false, "otherpath")).to eql(http_hostname_port + "/otherpath")
+        end
+      end
+    end
+
+    describe "an regular hostname" do
+      let(:hostname) { true_hostname }
+      include_examples("proper host handling")
+    end
+
+    describe "an ipv4 host" do
+      let(:hostname) { ipv4_hostname }
+      include_examples("proper host handling")
+    end
+
+    describe "an ipv6 host" do
+      let(:hostname) { ipv6_hostname }
+      include_examples("proper host handling")
+    end
+  end
+
+  describe "sniffing" do
+    let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options.merge(client_opts)) }
+    let(:transport) { client.client.transport }
+
+    before do
+      allow(transport).to receive(:reload_connections!)
+    end
+
+    context "with sniffing enabled" do
+      let(:client_opts) { {:sniffing => true, :sniffing_delay => 1 } }
+
+      after do
+        client.stop_sniffing!
+      end
+
+      it "should start the sniffer" do
+        expect(client.sniffer_thread).to be_a(Thread)
+      end
+
+      it "should periodically sniff the client" do
+        sleep 2
+        expect(transport).to have_received(:reload_connections!).at_least(:once)
+      end
+    end
+
+    context "with sniffing disabled" do
+      let(:client_opts) { {:sniffing => false} }
+
+      it "should not start the sniffer" do
+        expect(client.sniffer_thread).to be_nil
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch_proxy_spec.rb

@@ -0,0 +1,58 @@
+require_relative "../../../spec/es_spec_helper"
+require 'stud/temporary'
+require 'elasticsearch'
+require "logstash/outputs/elasticsearch"
+
+describe "Proxy option" do
+  let(:settings) {
+    {
+      "hosts" => "node01",
+      "proxy" => proxy
+    }
+  }
+  subject {
+    LogStash::Outputs::ElasticSearch.new(settings)
+  }
+
+  before do
+    allow(::Elasticsearch::Client).to receive(:new).with(any_args)
+  end
+
+  describe "valid configs" do
+    before do
+      subject.register
+    end
+
+    context "when specified as a string" do
+      let(:proxy) { "http://127.0.0.1:1234" }
+
+      it "should set the proxy to the exact value" do
+        expect(::Elasticsearch::Client).to have_received(:new) do |options|
+          expect(options[:transport_options][:proxy]).to eql(proxy)
+        end
+      end
+    end
+
+    context "when specified as a hash" do
+      let(:proxy) { {"hosts" => "127.0.0.1", "protocol" => "http"} }
+
+      it "should pass through the proxy values as symbols" do
+        expected = {:hosts => proxy["hosts"], :protocol => proxy["protocol"]}
+        expect(::Elasticsearch::Client).to have_received(:new) do |options|
+          expect(options[:transport_options][:proxy]).to eql(expected)
+        end
+      end
+    end
+  end
+
+  describe "invalid configs" do
+    let(:proxy) { ["bad", "stuff"] }
+
+    it "should have raised an exception" do
+      expect {
+        subject.register
+      }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -0,0 +1,227 @@
+require_relative "../../../spec/es_spec_helper"
+require "flores/random"
+require "logstash/outputs/elasticsearch"
+
+describe "outputs/elasticsearch" do
+  context "with an active instance" do
+    let(:options) {
+      {
+        "index" => "my-index",
+        "hosts" => ["localhost","localhost:9202"],
+        "path" => "some-path"
+      }
+    }
+
+    let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
+
+    let(:manticore_host) {
+      eso.client.send(:client).transport.options[:hosts].first
+    }
+
+    around(:each) do |block|
+      eso.register
+      block.call()
+      eso.close
+    end
+
+    describe "getting a document type" do
+      it "should default to 'logs'" do
+        expect(eso.send(:get_event_type, LogStash::Event.new)).to eql("logs")
+      end
+
+      it "should get the type from the event if nothing else specified in the config" do
+        expect(eso.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo")
+      end
+
+      context "with 'document type set'" do
+        let(:options) { super.merge("document_type" => "bar")}
+        it "should get the event type from the 'document_type' setting" do
+          expect(eso.send(:get_event_type, LogStash::Event.new())).to eql("bar")
+        end
+      end
+
+      context "with a bad type" do
+        let(:type_arg) { ["foo"] }
+        let(:result) { eso.send(:get_event_type, LogStash::Event.new("type" => type_arg)) }
+
+        before do
+          allow(eso.instance_variable_get(:@logger)).to receive(:warn)
+          result
+        end
+
+        it "should call @logger.warn and return nil" do
+          expect(eso.instance_variable_get(:@logger)).to have_received(:warn).with(/Bad event type!/, anything).once
+        end
+
+        it "should set the type to the stringified value" do
+          expect(result).to eql(type_arg.to_s)
+        end
+      end
+    end
+
+    describe "with path" do
+      it "should properly create a URI with the path" do
+        expect(eso.path).to eql(options["path"])
+      end
+
+      it "should properly set the path on the HTTP client adding slashes" do
+        expect(manticore_host).to include("/" + options["path"] + "/")
+      end
+
+      context "with extra slashes" do
+        let(:path) { "/slashed-path/ "}
+        let(:eso) {
+          LogStash::Outputs::ElasticSearch.new(options.merge("path" => "/some-path/"))
+        }
+
+        it "should properly set the path on the HTTP client without adding slashes" do
+          expect(manticore_host).to include(options["path"])
+        end
+      end
+    end
+    describe "without a port specified" do
+      it "should properly set the default port (9200) on the HTTP client" do
+        expect(manticore_host).to include("9200")
+      end
+    end
+    describe "with a port other than 9200 specified" do
+      let(:manticore_host) {
+        eso.client.send(:client).transport.options[:hosts].last
+      }
+      it "should properly set the specified port on the HTTP client" do
+        expect(manticore_host).to include("9202")
+      end
+    end
+
+    describe "#multi_receive" do
+      let(:events) { [double("one"), double("two"), double("three")] }
+      let(:events_tuples) { [double("one t"), double("two t"), double("three t")] }
+      let(:options) { super.merge("flush_size" => 2) }
+
+      before do
+        allow(eso).to receive(:retrying_submit).with(anything)
+        events.each_with_index do |e,i|
+          et = events_tuples[i]
+          allow(eso).to receive(:event_action_tuple).with(e).and_return(et)
+        end
+        eso.multi_receive(events)
+      end
+
+      it "should receive an array of events and invoke retrying_submit with them, split by flush_size" do
+        expect(eso).to have_received(:retrying_submit).with(events_tuples.slice(0,2))
+        expect(eso).to have_received(:retrying_submit).with(events_tuples.slice(2,3))
+      end
+
+    end
+
+  end
+
+  # TODO(sissel): Improve this. I'm not a fan of using message expectations (expect().to receive...)
+  # especially with respect to logging to verify a failure/retry has occurred. For now, this
+  # should suffice, though.
+  context "with timeout set" do
+    let(:listener) { Flores::Random.tcp_listener }
+    let(:port) { listener[2] }
+    let(:options) do
+      {
+        "manage_template" => false,
+        "hosts" => "localhost:#{port}",
+        "flush_size" => 1,
+        "timeout" => 0.1, # fast timeout
+      }
+    end
+    let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
+
+    before do
+      eso.register
+
+      # Expect a timeout to be logged.
+      expect(eso.logger).to receive(:error).with(/Attempted to send a bulk request/, anything)
+    end
+
+    after do
+      listener[0].close
+      # Stop the receive buffer, but don't flush because that would hang forever in this case since ES never returns a result
+      eso.instance_variable_get(:@buffer).stop(false,false)
+      eso.close
+    end
+
+    it "should fail after the timeout" do
+      Thread.new { eso.receive(LogStash::Event.new) }
+
+      # Allow the timeout to occur.
+      sleep(options["timeout"] + 0.5)
+    end
+  end
+
+  describe "the action option" do
+    subject(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
+    context "with a sprintf action" do
+      let(:options) { {"action" => "%{myactionfield}"} }
+
+      let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
+
+      it "should interpolate the requested action value when creating an event_action_tuple" do
+        expect(eso.event_action_tuple(event).first).to eql("update")
+      end
+    end
+
+    context "with an invalid action" do
+      let(:options) { {"action" => "SOME Garbaaage"} }
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+  end
+
+  describe "SSL end to end" do
+    shared_examples("an encrypted client connection") do
+      it "should enable SSL in manticore" do
+        expect(eso.client.client_options[:hosts].map {|h| URI.parse(h).scheme}.uniq).to eql(['https'])
+      end
+    end
+
+    let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
+    subject(:manticore) { eso.client.client}
+
+    before do
+      eso.register
+    end
+
+    context "With the 'ssl' option" do
+      let(:options) { {"ssl" => true}}
+
+      include_examples("an encrypted client connection")
+    end
+
+    context "With an https host" do
+      let(:options) { {"hosts" => "https://localhost"} }
+      include_examples("an encrypted client connection")
+    end
+  end
+
+  describe "retry_on_conflict" do
+    let(:num_retries) { 123 }
+    let(:event) { LogStash::Event.new("message" => "blah") }
+    subject(:eso) {LogStash::Outputs::ElasticSearch.new(options.merge('retry_on_conflict' => num_retries))}
+
+    context "with a regular index" do
+      let(:options) { {"action" => "index"} }
+
+      it "should interpolate the requested action value when creating an event_action_tuple" do
+        action, params, event_data = eso.event_action_tuple(event)
+        expect(params).not_to include({:_retry_on_conflict => num_retries})
+      end
+    end
+
+    context "using a plain update" do
+      let(:options) { {"action" => "update", "retry_on_conflict" => num_retries} }
+
+      it "should interpolate the requested action value when creating an event_action_tuple" do
+        action, params, event_data = eso.event_action_tuple(event)
+        expect(params).to include({:_retry_on_conflict => num_retries})
+      end
+    end
+  end
+end