logstash-output-elasticsearch 0.1.19-java → 0.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 88af0f35fb188226ed8a3af96835a57aef17cde6
-  data.tar.gz: 4e826e7a0dc993a21ca18a894d59fc2a73670f3d
+  metadata.gz: 6a7d8af0601791a0fa557ff0e61b995aaa514858
+  data.tar.gz: 7dfd959db33857c44ccc0085709d5e0967a87add
 SHA512:
-  metadata.gz: 542687e54823d84d895ac468a484ba80f76ff7f31909e424d26f065db1d5b167c20950f7003b6722b39ef9d6eb6aed9a546c5366ef1d249c5daf0b1fc992180b
-  data.tar.gz: f86b66b834d212bbd43ad77b48c273e1e960ff57c216c3b1129edfc1591d3377e660753d084f262cfc391a3999c1b8ae380a7a1cb42460fccb4109c20e384f7c
+  metadata.gz: 3c6af511fd72b689d3fedbae44198bf4f0b2823856289cc541905f8ae74b1929897a251deb2bc0b598cb809df3575d6d5c45f48dcee9706b7cd62b4a78f786f6
+  data.tar.gz: b0d2e8d2d12341ee0c975339e34b7dbfcb8614c1457ca59905fe3b7728159e73ca9f3071213e7c4b62cd2f3f9bc273ea0fdb0eb834549f76b6ba28bc64635764

data/lib/logstash/outputs/elasticsearch/elasticsearch-template.json

@@ -5,7 +5,7 @@
   },
   "mappings" : {
     "_default_" : {
-       "_all" : {"enabled" : true},
+       "_all" : {"enabled" : true, "omit_norms" : true},
        "dynamic_templates" : [ {
          "message_field" : {
            "match" : "message",
@@ -31,7 +31,6 @@
          "geoip"  : {
            "type" : "object",
              "dynamic": true,
-             "path": "full",
              "properties" : {
                "location" : { "type" : "geo_point" }
              }

data/lib/logstash/outputs/elasticsearch/protocol.rb

@@ -228,19 +228,23 @@ module LogStash::Outputs::Elasticsearch
           when "index"
             request = org.elasticsearch.action.index.IndexRequest.new(args[:_index])
             request.id(args[:_id]) if args[:_id]
+            request.routing(args[:_routing]) if args[:_routing]
             request.source(source)
           when "delete"
             request = org.elasticsearch.action.delete.DeleteRequest.new(args[:_index])
             request.id(args[:_id])
+            request.routing(args[:_routing]) if args[:_routing]
           when "create"
             request = org.elasticsearch.action.index.IndexRequest.new(args[:_index])
             request.id(args[:_id]) if args[:_id]
+            request.routing(args[:_routing]) if args[:_routing]
             request.source(source)
             request.opType("create")
           when "create_unless_exists"
             unless args[:_id].nil?
               request = org.elasticsearch.action.index.IndexRequest.new(args[:_index])
               request.id(args[:_id])
+              request.routing(args[:_routing]) if args[:_routing]
               request.source(source)
               request.opType("create")
             else

data/lib/logstash/outputs/elasticsearch.rb

@@ -81,7 +81,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
   # The document ID for the index. Useful for overwriting existing entries in
   # Elasticsearch with the same ID.
-  config :document_id, :validate => :string, :default => nil
+  config :document_id, :validate => :string
+
+  # A routing override to be applied to all processed events.
+  # This can be dynamic using the `%{foo}` syntax.
+  config :routing, :validate => :string
 
   # The name of your cluster if you set it on the Elasticsearch side. Useful
   # for discovery.
@@ -299,6 +303,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
     if @embedded
       raise(LogStash::ConfigurationError, "The 'embedded => true' setting is only valid for the elasticsearch output under JRuby. You are running #{RUBY_DESCRIPTION}") unless LogStash::Environment.jruby?
+      @logger.warn("The 'embedded => true' setting is enabled. This is not recommended for production use!!!")
 #      LogStash::Environment.load_elasticsearch_jars!
 
       # Default @host with embedded to localhost. This should help avoid
@@ -399,7 +404,8 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     index = event.sprintf(@index)
 
     document_id = @document_id ? event.sprintf(@document_id) : nil
-    buffer_receive([event.sprintf(@action), { :_id => document_id, :_index => index, :_type => type }, event])
+    routing = @routing ? event.sprintf(@routing) : nil
+    buffer_receive([event.sprintf(@action), { :_id => document_id, :_index => index, :_type => type, :_routing => routing }, event])
   end # def receive
 
   public

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '0.1.19'
+  s.version         = '0.2.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "Output events to elasticsearch"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Jar dependencies
-  s.requirements << "jar 'org.elasticsearch:elasticsearch', '1.4.0'"
+  s.requirements << "jar 'org.elasticsearch:elasticsearch', '1.5.0'"
 
   # Gem dependencies
   s.add_runtime_dependency 'concurrent-ruby'

data/spec/outputs/elasticsearch_spec.rb

@@ -73,6 +73,230 @@ describe "outputs/elasticsearch" do
     end
   end
 
+  describe "ship lots of events w/ default index_type and fixed routing key using http protocol", :elasticsearch => true do
+    # Generate a random index name
+    index = 10.times.collect { rand(10).to_s }.join("")
+    type = 10.times.collect { rand(10).to_s }.join("")
+
+    # Write 900 events so that we can verify these have been routed correctly.
+    event_count = 900
+    flush_size = rand(200) + 1
+
+    config <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch {
+          host => "127.0.0.1"
+          index => "#{index}"
+          flush_size => #{flush_size}
+          routing => "test"
+          protocol => "http"
+        }
+      }
+    CONFIG
+
+    agent do
+      # Try a few times to check if we have the correct number of events stored
+      # in ES.
+      #
+      # We try multiple times to allow final agent flushes as well as allowing
+      # elasticsearch to finish processing everything.
+      ftw = FTW::Agent.new
+      ftw.post!("http://localhost:9200/#{index}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+        response = ftw.get!("http://127.0.0.1:9200/#{index}/_count?q=*")
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        count = result["count"]
+        insist { count } == event_count
+      end
+
+      response = ftw.get!("http://127.0.0.1:9200/#{index}/_count?q=*&routing=test")
+      data = ""
+      response.read_body { |chunk| data << chunk }
+      result = LogStash::Json.load(data)
+      count = result["count"]
+      insist { count } == event_count
+    end
+  end
+
+describe "ship lots of events w/ default index_type and dynamic routing key using http protocol", :elasticsearch => true do
+    # Generate a random index name
+    index = 10.times.collect { rand(10).to_s }.join("")
+    type = 10.times.collect { rand(10).to_s }.join("")
+
+    # Write 900 events so that we can verify these have been routed correctly.
+    event_count = 900
+    flush_size = rand(200) + 1
+
+    config <<-CONFIG
+      input {
+        generator {
+          message => "test"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch {
+          host => "127.0.0.1"
+          index => "#{index}"
+          flush_size => #{flush_size}
+          routing => "%{message}"
+          protocol => "http"
+        }
+      }
+    CONFIG
+
+    agent do
+      # Try a few times to check if we have the correct number of events stored
+      # in ES.
+      #
+      # We try multiple times to allow final agent flushes as well as allowing
+      # elasticsearch to finish processing everything.
+      ftw = FTW::Agent.new
+      ftw.post!("http://localhost:9200/#{index}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+        response = ftw.get!("http://127.0.0.1:9200/#{index}/_count?q=*")
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        count = result["count"]
+        insist { count } == event_count
+      end
+
+      response = ftw.get!("http://127.0.0.1:9200/#{index}/_count?q=*&routing=test")
+      data = ""
+      response.read_body { |chunk| data << chunk }
+      result = LogStash::Json.load(data)
+      count = result["count"]
+      insist { count } == event_count
+    end
+  end
+
+  describe "ship lots of events w/ default index_type and fixed routing key using transport protocol", :elasticsearch => true do
+    # Generate a random index name
+    index = 10.times.collect { rand(10).to_s }.join("")
+    type = 10.times.collect { rand(10).to_s }.join("")
+
+    # Write 900 events so that we can verify these have been routed correctly.
+    event_count = 900
+    flush_size = rand(200) + 1
+
+    config <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch {
+          host => "127.0.0.1"
+          index => "#{index}"
+          flush_size => #{flush_size}
+          routing => "test"
+          protocol => "transport"
+        }
+      }
+    CONFIG
+
+    agent do
+      # Try a few times to check if we have the correct number of events stored
+      # in ES.
+      #
+      # We try multiple times to allow final agent flushes as well as allowing
+      # elasticsearch to finish processing everything.
+      ftw = FTW::Agent.new
+      ftw.post!("http://localhost:9200/#{index}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+        response = ftw.get!("http://127.0.0.1:9200/#{index}/_count?q=*")
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        count = result["count"]
+        insist { count } == event_count
+      end
+
+      response = ftw.get!("http://127.0.0.1:9200/#{index}/_count?q=*&routing=test")
+      data = ""
+      response.read_body { |chunk| data << chunk }
+      result = LogStash::Json.load(data)
+      count = result["count"]
+      insist { count } == event_count
+    end
+  end
+
+  describe "ship lots of events w/ default index_type and fixed routing key using node protocol", :elasticsearch => true do
+    # Generate a random index name
+    index = 10.times.collect { rand(10).to_s }.join("")
+    type = 10.times.collect { rand(10).to_s }.join("")
+
+    # Write 900 events so that we can verify these have been routed correctly.
+    event_count = 900
+    flush_size = rand(200) + 1
+
+    config <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch {
+          host => "127.0.0.1"
+          index => "#{index}"
+          flush_size => #{flush_size}
+          routing => "test"
+          protocol => "node"
+        }
+      }
+    CONFIG
+
+    agent do
+      # Try a few times to check if we have the correct number of events stored
+      # in ES.
+      #
+      # We try multiple times to allow final agent flushes as well as allowing
+      # elasticsearch to finish processing everything.
+      ftw = FTW::Agent.new
+      ftw.post!("http://localhost:9200/#{index}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+        response = ftw.get!("http://127.0.0.1:9200/#{index}/_count?q=*")
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        count = result["count"]
+        insist { count } == event_count
+      end
+
+      response = ftw.get!("http://127.0.0.1:9200/#{index}/_count?q=*&routing=test")
+      data = ""
+      response.read_body { |chunk| data << chunk }
+      result = LogStash::Json.load(data)
+      count = result["count"]
+      insist { count } == event_count
+    end
+  end
+
   describe "node client create actions", :elasticsearch => true do
     require "logstash/outputs/elasticsearch"
     require "elasticsearch"
@@ -475,9 +699,9 @@ describe "outputs/elasticsearch" do
   describe "failures in bulk class expected behavior", :elasticsearch => true do
     let(:template) { '{"template" : "not important, will be updated by :index"}' }
     let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
-    let(:action1) { ["index", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event1] }
+    let(:action1) { ["index", {:_id=>nil, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event1] }
     let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
-    let(:action2) { ["index", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event2] }
+    let(:action2) { ["index", {:_id=>nil, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event2] }
     let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
     let(:max_retries) { 3 }
 
@@ -486,8 +710,6 @@ describe "outputs/elasticsearch" do
         .any_instance.stub(:bulk).and_return(*resp)
       LogStash::Outputs::Elasticsearch::Protocols::NodeClient
         .any_instance.stub(:bulk).and_return(*resp)
-      LogStash::Outputs::Elasticsearch::Protocols::TransportClient
-        .any_instance.stub(:bulk).and_return(*resp)
     end
 
     ["node", "transport", "http"].each do |protocol|
@@ -539,7 +761,7 @@ describe "outputs/elasticsearch" do
           end
           subject.register
           subject.receive(event1)
-          subject.buffer_flush(:final => true)
+          subject.teardown
         end
 
         it "should retry actions with response status of 503" do
@@ -587,9 +809,10 @@ describe "outputs/elasticsearch" do
           subject.register
           subject.receive(invalid_event)
           expect(subject).not_to receive(:retry_push)
-          subject.buffer_flush(:final => true)
+          subject.teardown
 
           @es.indices.refresh
+          sleep(5)
           Stud::try(10.times) do
             r = @es.search
             insist { r["hits"]["total"] } == 0
@@ -600,9 +823,10 @@ describe "outputs/elasticsearch" do
           subject.register
           subject.receive(event1)
           expect(subject).not_to receive(:retry_push)
-          subject.buffer_flush(:final => true)
+          subject.teardown
 
           @es.indices.refresh
+          sleep(5)
           Stud::try(10.times) do
             r = @es.search
             insist { r["hits"]["total"] } == 1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.1.19
+  version: 0.2.0
 platform: java
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-03-26 00:00:00.000000000 Z
+date: 2015-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -235,7 +235,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- jar 'org.elasticsearch:elasticsearch', '1.4.0'
+- jar 'org.elasticsearch:elasticsearch', '1.5.0'
 rubyforge_project:
 rubygems_version: 2.1.9
 signing_key: