logstash-output-elasticsearch 2.1.5-java → 2.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e1c06d07d392d4fba071aa01553091dd85461b3b
-  data.tar.gz: 368990ab9b1464f955418125388028940e8576aa
+  metadata.gz: 0bacdce326400468748dde5d68f0d93612c481d0
+  data.tar.gz: ed36edd4dc675f9c5940f34314dc511134e3b0db
 SHA512:
-  metadata.gz: 221ea508978c3bef5e9f03794b4af02d7e96234b60b7b2b1cf3f8b18a263b90e8bb1ef8d4a372bd1f6d376ccefe65d54fca7272b573a5762011e116057051506
-  data.tar.gz: daa946bd260816118bb2e88aa3173220f0999352417bc00af7974fc8b21365f1e1218096fb2e6430c987e04b0a48175b93139bb908c9cac52d672e5f2b68df55
+  metadata.gz: c1eefd941d00fcef32a5194f0bb8b457f8e8bb690cec751c4c347a89c655ea0ea4028addd7211d36e49fccda107fcbe569ed64fb5ea1869f93e0cd4738612946
+  data.tar.gz: 13c2da8b12fa91911e720080d75c3d0afbc984a21ce4db3b4ce150159289db63189e4455eda63005e5c170c30e59c4f6bcb5ba7611a77d465be8319f054ffb63

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.1.6
+ - Serialize access to the connection pool in es-ruby client
+ - Add support for parent relationship
+
 ## 2.1.5
  - Sprintf style 'action' parameters no longer raise a LogStash::ConfigurationError
 

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -132,6 +132,7 @@ module LogStash; module Outputs; class ElasticSearch;
         :_routing => @routing ? event.sprintf(@routing) : nil
       }
 
+      params[:parent] = event.sprintf(@parent) if @parent
       params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @action == 'update' && @upsert != ""
       params
     end

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -56,6 +56,10 @@ module LogStash; module Outputs; class ElasticSearch
       # This can be dynamic using the `%{foo}` syntax.
       mod.config :routing, :validate => :string
 
+      # For child documents, ID of the associated parent.
+      # This can be dynamic using the `%{foo}` syntax.
+      mod.config :parent, :validate => :string, :default => nil
+
       # Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
       # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
       #     `"127.0.0.1"`

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -18,18 +18,27 @@ module LogStash; module Outputs; class ElasticSearch;
       # @options = DEFAULT_OPTIONS.merge(options)
       @options = options
       @client = build_client(@options)
+      # mutex to prevent requests and sniffing to access the
+      # connection pool at the same time
+      @request_mutex = Mutex.new
       start_sniffing!
     end
 
     def template_install(name, template, force=false)
-      if template_exists?(name) && !force
-        @logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name)
-        return
+      @request_mutex.synchronize do
+        if template_exists?(name) && !force
+          @logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name)
+          return
+        end
+        template_put(name, template)
       end
-      template_put(name, template)
     end
 
     def bulk(actions)
+      @request_mutex.synchronize { non_threadsafe_bulk(actions) }
+    end
+
+    def non_threadsafe_bulk(actions)
       return if actions.empty?
       bulk_body = actions.collect do |action, args, source|
         if action == 'update'
@@ -61,7 +70,7 @@ module LogStash; module Outputs; class ElasticSearch;
       if options[:sniffing]
         @sniffer_thread = Thread.new do
           loop do
-            sniff!
+            @request_mutex.synchronize { sniff! }
             sleep (options[:sniffing_delay].to_f || 30)
           end
         end

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '2.1.5'
+  s.version         = '2.2.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "Output events to elasticsearch"

data/spec/integration/outputs/parent_spec.rb

@@ -0,0 +1,73 @@
+require_relative "../../../spec/es_spec_helper"
+
+shared_examples "a parent indexer" do
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { 10000 + rand(500) }
+    let(:flush_size) { rand(200) + 1 }
+    let(:parent) { "not_implemented" }
+    let(:config) { "not_implemented" }
+    subject { LogStash::Outputs::ElasticSearch.new(config) }
+
+    before do
+      # Add mapping and a parent document
+      index_url = "http://#{get_host_port()}/#{index}"
+      ftw = FTW::Agent.new
+      mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
+      ftw.put!("#{index_url}", :body => mapping.to_json)
+      pdoc = { "foo" => "bar" }
+      ftw.put!("#{index_url}/#{type}_parent/test", :body => pdoc.to_json)
+      
+      subject.register
+      event_count.times do
+        subject.receive(LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type))
+      end
+    end
+
+
+    it "ships events" do
+      index_url = "http://#{get_host_port()}/#{index}"
+
+      ftw = FTW::Agent.new
+      ftw.post!("#{index_url}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
+        data = ""
+        response = ftw.post!("#{index_url}/_count?q=*", :body => query.to_json)
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        cur_count = result["count"]
+        insist { cur_count } == event_count
+      end
+    end
+end
+
+describe "(http protocol) index events with static parent", :integration => true do
+  it_behaves_like 'a parent indexer' do
+    let(:parent) { "test" }
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index,
+        "flush_size" => flush_size,
+        "parent" => parent
+      }
+    }
+  end
+end
+
+describe "(http_protocol) index events with fieldref in parent value", :integration => true do
+  it_behaves_like 'a parent indexer' do
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index,
+        "flush_size" => flush_size,
+        "parent" => "%{link_to}"
+      }
+    }
+  end
+end
+

data/spec/unit/outputs/elasticsearch/protocol_spec.rb

@@ -25,7 +25,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
       it "should periodically sniff the client" do
         sleep 2
-        expect(transport).to have_received(:reload_connections!)
+        expect(transport).to have_received(:reload_connections!).at_least(:once)
       end
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.1.5
+  version: 2.2.0
 platform: java
 authors:
 - Elastic
@@ -206,6 +206,7 @@ files:
 - spec/es_spec_helper.rb
 - spec/integration/outputs/create_spec.rb
 - spec/integration/outputs/index_spec.rb
+- spec/integration/outputs/parent_spec.rb
 - spec/integration/outputs/retry_spec.rb
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/secure_spec.rb
@@ -247,6 +248,7 @@ test_files:
 - spec/es_spec_helper.rb
 - spec/integration/outputs/create_spec.rb
 - spec/integration/outputs/index_spec.rb
+- spec/integration/outputs/parent_spec.rb
 - spec/integration/outputs/retry_spec.rb
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/secure_spec.rb