logstash-output-elasticsearch 0.1.6 → 3.0.0

data/spec/integration/outputs/routing_spec.rb

@@ -0,0 +1,65 @@
+require_relative "../../../spec/es_spec_helper"
+
+shared_examples "a routing indexer" do
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { 10000 + rand(500) }
+    let(:flush_size) { rand(200) + 1 }
+    let(:routing) { "not_implemented" }
+    let(:config) { "not_implemented" }
+    subject { LogStash::Outputs::ElasticSearch.new(config) }
+
+    before do
+      subject.register
+      event_count.times do
+        subject.receive(LogStash::Event.new("message" => "Hello World!", "type" => type))
+      end
+    end
+
+
+    it "ships events" do
+      index_url = "http://#{get_host_port()}/#{index}"
+
+      ftw = FTW::Agent.new
+      ftw.post!("#{index_url}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+        response = ftw.get!("#{index_url}/_count?q=*&routing=#{routing}")
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        cur_count = result["count"]
+        insist { cur_count } == event_count
+      end
+    end
+end
+
+describe "(http protocol) index events with static routing", :integration => true do
+  it_behaves_like 'a routing indexer' do
+    let(:routing) { "test" }
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index,
+        "flush_size" => flush_size,
+        "routing" => routing
+      }
+    }
+  end
+end
+
+describe "(http_protocol) index events with fieldref in routing value", :integration => true do
+  it_behaves_like 'a routing indexer' do
+    let(:routing) { "test" }
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index,
+        "flush_size" => flush_size,
+        "routing" => "%{message}"
+      }
+    }
+  end
+end
+

data/spec/integration/outputs/secure_spec.rb

@@ -0,0 +1,108 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "node_name" => "logstash",
+      "cluster" => "elasticsearch",
+      "hosts" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+      "ssl" => true,
+      "cacert" => "/tmp/ca/certs/cacert.pem",
+      # or
+      #"truststore" => "/tmp/ca/truststore.jks",
+      #"truststore_password" => "testeteste"
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.flush
+    }.to_not raise_error
+  end
+end
+
+describe "connect using HTTP Authentication", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "cluster" => "elasticsearch",
+      "hosts" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.flush
+    }.to_not raise_error
+  end
+end
+
+describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "node_name" => "logstash",
+      "cluster" => "elasticsearch",
+      "hosts" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+      "ssl" => true,
+      "cacert" => "/tmp/ca/certs/cacert.pem",
+      # or
+      #"truststore" => "/tmp/ca/truststore.jks",
+      #"truststore_password" => "testeteste"
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.flush
+    }.to_not raise_error
+  end
+end
+
+describe "connect using HTTP Authentication", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "hosts" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.flush
+    }.to_not raise_error
+  end
+end

data/spec/integration/outputs/templates_spec.rb

@@ -0,0 +1,90 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "index template expected behavior", :integration => true do
+  subject! do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "manage_template" => true,
+      "template_overwrite" => true,
+      "hosts" => "#{get_host_port()}"
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    # Delete all templates first.
+    require "elasticsearch"
+
+    # Clean ES of data before we start.
+    @es = get_client
+    @es.indices.delete_template(:name => "*")
+
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+
+    subject.register
+
+    subject.receive(LogStash::Event.new("message" => "sample message here"))
+    subject.receive(LogStash::Event.new("somevalue" => 100))
+    subject.receive(LogStash::Event.new("somevalue" => 10))
+    subject.receive(LogStash::Event.new("somevalue" => 1))
+    subject.receive(LogStash::Event.new("country" => "us"))
+    subject.receive(LogStash::Event.new("country" => "at"))
+    subject.receive(LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }))
+    subject.flush
+    @es.indices.refresh
+
+    # Wait or fail until everything's indexed.
+    Stud::try(20.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 7
+    end
+  end
+
+  it "permits phrase searching on string fields" do
+    results = @es.search(:q => "message:\"sample message\"")
+    insist { results["hits"]["total"] } == 1
+    insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
+  end
+
+  it "numbers dynamically map to a numeric type and permit range queries" do
+    results = @es.search(:q => "somevalue:[5 TO 105]")
+    insist { results["hits"]["total"] } == 2
+
+    values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+    insist { values }.include?(10)
+    insist { values }.include?(100)
+    reject { values }.include?(1)
+  end
+
+  it "does not create .raw field for the message field" do
+    results = @es.search(:q => "message.raw:\"sample message here\"")
+    insist { results["hits"]["total"] } == 0
+  end
+
+  it "creates .raw field from any string field which is not_analyzed" do
+    results = @es.search(:q => "country.raw:\"us\"")
+    insist { results["hits"]["total"] } == 1
+    insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
+
+    # partial or terms should not work.
+    results = @es.search(:q => "country.raw:\"u\"")
+    insist { results["hits"]["total"] } == 0
+  end
+
+  it "make [geoip][location] a geo_point" do
+    results = @es.search(:body => { "query" => { "bool" => { "must" => { "match_all" => {} }, "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } } } })
+    insist { results["hits"]["total"] } == 1
+    insist { results["hits"]["hits"][0]["_source"]["geoip"]["location"] } == [ 0.0, 0.0 ]
+  end
+
+  it "aggregate .raw results correctly " do
+    results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.raw" } } } })["aggregations"]["my_agg"]
+    terms = results["buckets"].collect { |b| b["key"] }
+
+    insist { terms }.include?("us")
+
+    # 'at' is a stopword, make sure stopwords are not ignored.
+    insist { terms }.include?("at")
+  end
+end

data/spec/integration/outputs/update_spec.rb

@@ -0,0 +1,188 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "Update actions", :integration => true do
+  require "logstash/outputs/elasticsearch"
+  require "elasticsearch"
+
+  def get_es_output( options={} )
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-update",
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "action" => "update"
+    }
+    LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    @es.index(
+      :index => 'logstash-update',
+      :type => 'logs',
+      :id => "123",
+      :body => { :message => 'Test', :counter => 1 }
+    )
+    @es.indices.refresh
+  end
+
+  it "should fail without a document_id" do
+    subject = get_es_output
+    expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+  end
+
+  context "when update only" do
+    it "should not create new document" do
+      subject = get_es_output({ 'document_id' => "456" } )
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.flush
+      expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+    end
+
+    it "should update existing document" do
+      subject = get_es_output({ 'document_id' => "123" })
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "updated message here"))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["message"] } == 'updated message here'
+    end
+
+    # The es ruby client treats the data field differently. Make sure this doesn't
+    # raise an exception
+    it "should update an existing document that has a 'data' field" do
+      subject = get_es_output({ 'document_id' => "123" })
+      subject.register
+      subject.receive(LogStash::Event.new("data" => "updated message here", "message" => "foo"))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["data"] } == 'updated message here'
+      insist { r["_source"]["message"] } == 'foo'
+    end
+  end
+
+  context "when using script" do
+    it "should increment a counter with event/doc 'count' variable" do
+      subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
+      subject.register
+      subject.receive(LogStash::Event.new("count" => 2))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 3
+    end
+
+    it "should increment a counter with event/doc '[data][count]' nested variable" do
+      subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
+      subject.register
+      subject.receive(LogStash::Event.new("data" => { "count" => 3 }))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 4
+    end
+
+    it "should increment a counter with event/doc 'count' variable with inline script" do
+      subject = get_es_output({
+        'document_id' => "123",
+        'script' => 'ctx._source.counter += event["counter"]',
+        'script_lang' => 'groovy',
+        'script_type' => 'inline'
+      })
+      subject.register
+      subject.receive(LogStash::Event.new("counter" => 3 ))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 4
+    end
+
+    it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
+      subject = get_es_output({
+        'document_id' => "123",
+        'doc_as_upsert' => true,
+        'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["counter"]; } else { ctx._source.counter = event["counter"]; }',
+        'script_lang' => 'groovy',
+        'script_type' => 'inline'
+      })
+      subject.register
+      subject.receive(LogStash::Event.new("counter" => 3 ))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 4
+    end
+
+    it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
+      subject = get_es_output({
+        'document_id' => "456",
+        'doc_as_upsert' => true,
+        'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["count"]; } else { ctx._source.counter = event["count"]; }',
+        'script_lang' => 'groovy',
+        'script_type' => 'inline'
+      })
+      subject.register
+      subject.receive(LogStash::Event.new("counter" => 3 ))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["counter"] } == 3
+    end
+
+    it "should increment a counter with event/doc 'count' variable with indexed script" do
+      @es.put_script lang: 'groovy', id: 'indexed_update', body: { script: 'ctx._source.counter += event["count"]' }
+      subject = get_es_output({
+        'document_id' => "123",
+        'script' => 'indexed_update',
+        'script_lang' => 'groovy',
+        'script_type' => 'indexed'
+      })
+      subject.register
+      subject.receive(LogStash::Event.new("count" => 4 ))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 5
+    end
+  end
+
+  context "when update with upsert" do
+    it "should create new documents with provided upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["message"] } == 'upsert message'
+    end
+
+    it "should create new documents with event/doc as upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["message"] } == 'sample message here'
+    end
+
+    context "when using script" do
+      it "should create new documents with upsert content" do
+        subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.flush
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["message"] } == 'upsert message'
+      end
+
+      it "should create new documents with event/doc as script params" do
+        subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
+        subject.register
+        subject.receive(LogStash::Event.new("counter" => 1))
+        subject.flush
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["counter"] } == 1
+      end
+    end
+  end
+end