logstash-output-elasticsearch 0.1.6 → 3.0.0

data/spec/es_spec_helper.rb

@@ -0,0 +1,77 @@
+require "logstash/devutils/rspec/spec_helper"
+require "ftw"
+require "logstash/plugin"
+require "logstash/json"
+require "stud/try"
+require "longshoreman"
+require "logstash/outputs/elasticsearch"
+
+CONTAINER_NAME = "logstash-output-elasticsearch-#{rand(999).to_s}"
+CONTAINER_IMAGE = "elasticsearch"
+CONTAINER_TAG = "2.0"
+
+DOCKER_INTEGRATION = ENV["DOCKER_INTEGRATION"]
+
+module ESHelper
+  def get_host_port
+    addr = DOCKER_INTEGRATION ? Longshoreman.new.get_host_ip : "127.0.0.1"
+    "#{addr}:#{get_port}"
+  end
+
+  def get_port
+    return 9200 unless DOCKER_INTEGRATION
+
+    container = Longshoreman::Container.new
+    container.get(CONTAINER_NAME)
+    container.rport(9200)
+  end
+
+  def get_client
+    Elasticsearch::Client.new(:hosts => [get_host_port])
+  end
+end
+
+
+RSpec.configure do |config|
+  config.include ESHelper
+
+  if DOCKER_INTEGRATION
+    # this :all hook gets run before every describe block that is tagged with :integration => true.
+    config.before(:all, :integration => true) do
+
+
+      # check if container exists already before creating new one.
+      begin
+        ls = Longshoreman::new
+        ls.container.get(CONTAINER_NAME)
+      rescue Docker::Error::NotFoundError
+        scriptDir = File.expand_path File.dirname(__FILE__) + '/fixtures/scripts'
+        Longshoreman.new("#{CONTAINER_IMAGE}:#{CONTAINER_TAG}", CONTAINER_NAME, {
+          'Cmd' => [ "-Des.script.inline=on", "-Des.script.indexed=on" ],
+          'HostConfig' => {
+            'Binds' => ["#{scriptDir}:/usr/share/elasticsearch/config/scripts"],
+            'PublishAllPorts' => true
+          }
+        })
+        # TODO(talevy): verify ES is running instead of static timeout
+        sleep 10
+      end
+    end
+
+    # we want to do a final cleanup after all :integration runs,
+    # but we don't want to clean up before the last block.
+    # This is a final blind check to see if the ES docker container is running and
+    # needs to be cleaned up. If no container can be found and/or docker is not
+    # running on the system, we do nothing.
+    config.after(:suite) do
+      # only cleanup docker container if system has docker and the container is running
+      begin
+        ls = Longshoreman::new
+        ls.container.get(CONTAINER_NAME)
+        ls.cleanup
+      rescue Docker::Error::NotFoundError, Excon::Errors::SocketError
+        # do nothing
+      end
+    end
+  end
+end

data/spec/fixtures/scripts/scripted_update.groovy

@@ -0,0 +1,2 @@
+ctx._source.counter += event["count"]
+

data/spec/fixtures/scripts/scripted_update_nested.groovy

@@ -0,0 +1,2 @@
+ctx._source.counter += event["data"]["count"]
+

data/spec/fixtures/scripts/scripted_upsert.groovy

@@ -0,0 +1,2 @@
+ctx._source.counter = event["counter"]
+

data/spec/integration/outputs/create_spec.rb

@@ -0,0 +1,55 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "client create actions", :integration => true do
+  require "logstash/outputs/elasticsearch"
+  require "elasticsearch"
+
+  def get_es_output(action, id = nil)
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-create",
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "action" => action
+    }
+    settings['document_id'] = id unless id.nil?
+    LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+  end
+
+  context "when action => create" do
+    it "should create new documents with or without id" do
+      subject = get_es_output("create", "id123")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.flush
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+
+    it "should create new documents without id" do
+      subject = get_es_output("create")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.flush
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+  end
+end

data/spec/integration/outputs/index_spec.rb

@@ -0,0 +1,68 @@
+require_relative "../../../spec/es_spec_helper"
+
+shared_examples "an indexer" do
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { 10000 + rand(500) }
+    let(:flush_size) { rand(200) + 1 }
+    let(:config) { "not implemented" }
+    subject { LogStash::Outputs::ElasticSearch.new(config) }
+
+    before do
+      subject.register
+      event_count.times do
+        subject.receive(LogStash::Event.new("message" => "Hello World!", "type" => type))
+      end
+    end
+
+    it "ships events" do
+      index_url = "http://#{get_host_port}/#{index}"
+
+      ftw = FTW::Agent.new
+      ftw.post!("#{index_url}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+        response = ftw.get!("#{index_url}/_count?q=*")
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        cur_count = result["count"]
+        insist { cur_count } == event_count
+      end
+
+      response = ftw.get!("#{index_url}/_search?q=*&size=1000")
+      data = ""
+      response.read_body { |chunk| data << chunk }
+      result = LogStash::Json.load(data)
+      result["hits"]["hits"].each do |doc|
+        insist { doc["_type"] } == type
+        insist { doc["_index"] } == index
+      end
+    end
+end
+
+describe "an indexer with custom index_type", :integration => true do
+  it_behaves_like "an indexer" do
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index,
+        "flush_size" => flush_size
+      }
+    }
+  end
+end
+
+describe "an indexer with no type value set (default to logs)", :integration => true do
+  it_behaves_like "an indexer" do
+    let(:type) { "logs" }
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index,
+        "flush_size" => flush_size
+      }
+    }
+  end
+end

data/spec/integration/outputs/parent_spec.rb

@@ -0,0 +1,73 @@
+require_relative "../../../spec/es_spec_helper"
+
+shared_examples "a parent indexer" do
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { 10000 + rand(500) }
+    let(:flush_size) { rand(200) + 1 }
+    let(:parent) { "not_implemented" }
+    let(:config) { "not_implemented" }
+    subject { LogStash::Outputs::ElasticSearch.new(config) }
+
+    before do
+      # Add mapping and a parent document
+      index_url = "http://#{get_host_port()}/#{index}"
+      ftw = FTW::Agent.new
+      mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
+      ftw.put!("#{index_url}", :body => mapping.to_json)
+      pdoc = { "foo" => "bar" }
+      ftw.put!("#{index_url}/#{type}_parent/test", :body => pdoc.to_json)
+      
+      subject.register
+      event_count.times do
+        subject.receive(LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type))
+      end
+    end
+
+
+    it "ships events" do
+      index_url = "http://#{get_host_port()}/#{index}"
+
+      ftw = FTW::Agent.new
+      ftw.post!("#{index_url}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
+        data = ""
+        response = ftw.post!("#{index_url}/_count?q=*", :body => query.to_json)
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        cur_count = result["count"]
+        insist { cur_count } == event_count
+      end
+    end
+end
+
+describe "(http protocol) index events with static parent", :integration => true do
+  it_behaves_like 'a parent indexer' do
+    let(:parent) { "test" }
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index,
+        "flush_size" => flush_size,
+        "parent" => parent
+      }
+    }
+  end
+end
+
+describe "(http_protocol) index events with fieldref in parent value", :integration => true do
+  it_behaves_like 'a parent indexer' do
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index,
+        "flush_size" => flush_size,
+        "parent" => "%{link_to}"
+      }
+    }
+  end
+end
+

data/spec/integration/outputs/pipeline_spec.rb

@@ -0,0 +1,75 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "Ingest pipeline execution behavior", :integration => true, :version_5x => true do
+  subject! do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "hosts" => "#{get_host_port()}",
+      "pipeline" => "apache-logs"
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  let(:ftw_client) { FTW::Agent.new }
+  let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
+  let(:apache_logs_pipeline) { '
+    {
+      "description" : "Pipeline to parse Apache logs",
+      "processors" : [
+        {
+          "grok": {
+            "field": "message",
+            "pattern": "%{COMBINEDAPACHELOG}"
+          }
+        }
+      ]
+    }'
+  }
+
+  before :each do
+    # Delete all templates first.
+    require "elasticsearch"
+
+    # Clean ES of data before we start.
+    @es = get_client
+    @es.indices.delete_template(:name => "*")
+
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+
+    # delete existing ingest pipeline
+    req = ftw_client.delete(ingest_url)
+    ftw_client.execute(req)
+
+    # register pipeline
+    req = ftw_client.put(ingest_url, :body => apache_logs_pipeline)
+    ftw_client.execute(req)
+
+    #TODO: Use esclient
+    #@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
+
+    subject.register
+    subject.receive(LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"'))
+    subject.flush
+    @es.indices.refresh
+
+    #Wait or fail until everything's indexed.
+    Stud::try(20.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 1
+    end
+  end
+
+  it "indexes using the proper pipeline" do
+    results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
+    insist { results["hits"]["total"] } == 1
+    insist { results["hits"]["hits"][0]["_source"]["response"] } == "200"
+    insist { results["hits"]["hits"][0]["_source"]["bytes"] } == "182"
+    insist { results["hits"]["hits"][0]["_source"]["verb"] } == "GET"
+    insist { results["hits"]["hits"][0]["_source"]["request"] } == "/scripts/netcat-webserver"
+    insist { results["hits"]["hits"][0]["_source"]["auth"] } == "-"
+    insist { results["hits"]["hits"][0]["_source"]["ident"] } == "-"
+    insist { results["hits"]["hits"][0]["_source"]["clientip"] } == "183.60.215.50"
+    insist { results["hits"]["hits"][0]["_source"]["junkfieldaaaa"] } == nil
+  end
+end

data/spec/integration/outputs/retry_spec.rb

@@ -0,0 +1,163 @@
+require "logstash/outputs/elasticsearch"
+require_relative "../../../spec/es_spec_helper"
+
+describe "failures in bulk class expected behavior", :integration => true do
+  let(:template) { '{"template" : "not important, will be updated by :index"}' }
+  let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
+  let(:action1) { ["index", {:_id=>nil, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event1] }
+  let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
+  let(:action2) { ["index", {:_id=>nil, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event2] }
+  let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
+
+  def mock_actions_with_response(*resp)
+    expanded_responses = resp.map do |resp|
+      items = resp["statuses"] && resp["statuses"].map do |status|
+        {"create" => {"status" => status, "error" => "Error for #{status}"}}
+      end
+
+      {
+        "errors" => resp["errors"],
+        "items" => items
+      }
+    end
+
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient).to receive(:bulk).and_return(*expanded_responses)
+  end
+
+  subject! do
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-2014.11.17",
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "retry_max_items" => 10,
+      "retry_max_interval" => 1,
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    # Delete all templates first.
+    require "elasticsearch"
+
+    # Clean ES of data before we start.
+    @es = get_client
+    @es.indices.delete_template(:name => "*")
+    @es.indices.delete(:index => "*")
+    @es.indices.refresh
+  end
+
+  after :each do
+    subject.close
+  end
+
+  it "should return no errors if all bulk actions are successful" do
+    mock_actions_with_response({"errors" => false})
+    expect(subject).to receive(:submit).with([action1, action2]).once.and_call_original
+    subject.register
+    subject.receive(event1)
+    subject.receive(event2)
+    subject.flush
+    sleep(2)
+  end
+
+  it "retry exceptions within the submit body" do
+    call_count = 0
+    subject.register
+
+    expect(subject.client).to receive(:bulk).with(anything).exactly(3).times do
+      if (call_count += 1) <= 2
+        raise "error first two times"
+      else
+        {"errors" => false}
+      end
+    end
+
+    subject.receive(event1)
+    subject.flush
+  end
+
+  it "should retry actions with response status of 503" do
+    mock_actions_with_response({"errors" => true, "statuses" => [200, 200, 503, 503]},
+                               {"errors" => true, "statuses" => [200, 503]},
+                               {"errors" => false})
+    expect(subject).to receive(:submit).with([action1, action1, action1, action2]).ordered.once.and_call_original
+    expect(subject).to receive(:submit).with([action1, action2]).ordered.once.and_call_original
+    expect(subject).to receive(:submit).with([action2]).ordered.once.and_call_original
+
+    subject.register
+    subject.receive(event1)
+    subject.receive(event1)
+    subject.receive(event1)
+    subject.receive(event2)
+    subject.flush
+    sleep(3)
+  end
+
+  it "should retry actions with response status of 429" do
+    subject.register
+
+    mock_actions_with_response({"errors" => true, "statuses" => [429]},
+                               {"errors" => false})
+    expect(subject).to receive(:submit).with([action1]).twice.and_call_original
+
+    subject.receive(event1)
+    subject.flush
+    sleep(3)
+  end
+
+  it "should retry an event infinitely until a non retryable status occurs" do
+    mock_actions_with_response({"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [500]})
+    expect(subject).to receive(:submit).with([action1]).exactly(6).times.and_call_original
+    subject.register
+    subject.receive(event1)
+    subject.flush
+    sleep(5)
+  end
+
+  it "non-retryable errors like mapping errors (400) should be dropped and not be retried (unfortunately)" do
+    subject.register
+    subject.receive(invalid_event)
+    expect(subject).to receive(:submit).once.and_call_original
+    subject.close
+
+    @es.indices.refresh
+    sleep(5)
+    Stud::try(10.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 0
+    end
+  end
+
+  it "successful requests should not be appended to retry queue" do
+    subject.register
+    subject.receive(event1)
+    expect(subject).to receive(:submit).once.and_call_original
+    subject.close
+    @es.indices.refresh
+    sleep(5)
+    Stud::try(10.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 1
+    end
+  end
+
+  it "should only index proper events" do
+    subject.register
+    subject.receive(invalid_event)
+    subject.receive(event1)
+    subject.close
+
+    @es.indices.refresh
+    sleep(5)
+    Stud::try(10.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 1
+    end
+  end
+end