logstash-output-elasticsearch 6.1.0-java → 6.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f8df743661ac7aed2fb52dddaf677ce1705a0612
-  data.tar.gz: 10bcb6b88076eafe00aea65c6d9e01d4d3fd8648
+  metadata.gz: 8d8fa5b44c2bf968c6460d25f27caf4f1b814467
+  data.tar.gz: 3225315b016f698fcb2a29f48f180d79f8ae15ab
 SHA512:
-  metadata.gz: 605c1958b1b4d22e51132e71932174c86c3d1696b80a7f35e07ec147301072dec9befeb64fae5bd240cff7c38f791fa3ede8bfa8a4781cd705a956d4a6e5b6b4
-  data.tar.gz: 69fae6a096cfa00b41037800d7a09737d0fec46b9722796f746d2aa1074be49147b3d2e8f2ec9ca39fdabf35130a2d603f8f512f771bceb09b1afdc1ffa6d0e8
+  metadata.gz: 55d2b2af8b3e15a814cfc2e592251488eadb8325a6b075976af4199f98df79b0e8d70ba3c7e27352a6f69c7e741255fd3b46b523efbbb1cd6c32a3f4e07847bb
+  data.tar.gz: 18268aa5143178e89467b90f81c278cd449f49ba8b5e03d2a00ec7a4e3f5a05885ccba70252de25c4ebfc3e8137e9e28b3792a6af71b3de4cbf4665d0a642308

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 6.2.0
+- Add version number / version conflict support
+
 ## 6.1.0
 - Add option to use an absolute healthcheck path
 

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -6,6 +6,13 @@ module LogStash; module Outputs; class ElasticSearch;
 
     RETRYABLE_CODES = [429, 503]
     SUCCESS_CODES = [200, 201]
+    CONFLICT_CODE = 409
+
+    # When you use external versioning, you are communicating that you want
+    # to ignore conflicts. More obviously, since an external version is a 
+    # constant part of the incoming document, we should not retry, as retrying
+    # will never succeed. 
+    VERSION_TYPES_PERMITTING_CONFLICT = ["external", "external_gt", "external_gte"]
 
     def register
       @stopping = Concurrent::AtomicBoolean.new(false)
@@ -120,9 +127,13 @@ module LogStash; module Outputs; class ElasticSearch;
         status = action_props["status"]
         failure  = action_props["error"]
         action = actions[idx]
+        action_params = action[1]
 
         if SUCCESS_CODES.include?(status)
           next
+        elsif CONFLICT_CODE == status && VERSION_TYPES_PERMITTING_CONFLICT.include?(action_params[:version_type])
+          @logger.debug "Ignoring external version conflict: status[#{status}] failure[#{failure}] version[#{action_params[:version]}] version_type[#{action_params[:version_type]}]"
+          next
         elsif RETRYABLE_CODES.include?(status)
           @logger.info "retrying failed action with response code: #{status} (#{failure})"
           actions_to_retry << action
@@ -159,6 +170,14 @@ module LogStash; module Outputs; class ElasticSearch;
         params[:_retry_on_conflict] = @retry_on_conflict
       end
 
+      if @version
+        params[:version] = event.sprintf(@version)
+      end
+
+      if @version_type
+        params[:version_type] = event.sprintf(@version_type)
+      end
+
       params
     end
 

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -61,6 +61,15 @@ module LogStash; module Outputs; class ElasticSearch
       # Elasticsearch with the same ID.
       mod.config :document_id, :validate => :string
 
+      # The version to use for indexing. Use sprintf syntax like `%{my_version}` to use a field value here.
+      # See https://www.elastic.co/blog/elasticsearch-versioning-support.
+      mod.config :version, :validate => :string
+      
+      # The version_type to use for indexing.
+      # See https://www.elastic.co/blog/elasticsearch-versioning-support.
+      # See also https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#_version_types
+      mod.config :version_type, :validate => ["internal", 'external', "external_gt", "external_gte", "force"]
+
       # A routing override to be applied to all processed events.
       # This can be dynamic using the `%{foo}` syntax.
       mod.config :routing, :validate => :string

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -36,6 +36,20 @@ module LogStash; module Outputs; class ElasticSearch;
       client_settings.merge! setup_ssl(logger, params)
       common_options.merge! setup_basic_auth(logger, params)
 
+      external_version_types = ["external", "external_gt", "external_gte"]
+      # External Version validation
+      raise(
+        LogStash::ConfigurationError,
+        "External versioning requires the presence of a version number."
+      ) if external_version_types.include?(params.fetch('version_type', '')) and params.fetch("version", nil) == nil
+ 
+
+      # Create API setup
+      raise(
+        LogStash::ConfigurationError,
+        "External versioning is not supported by the create action."
+      ) if params['action'] == 'create' and external_version_types.include?(params.fetch('version_type', ''))
+
       # Update API setup
       raise( LogStash::ConfigurationError,
         "doc_as_upsert and scripted_upsert are mutually exclusive."
@@ -46,6 +60,11 @@ module LogStash; module Outputs; class ElasticSearch;
         "Specifying action => 'update' needs a document_id."
       ) if params['action'] == 'update' and params.fetch('document_id', '') == ''
 
+      raise(
+        LogStash::ConfigurationError,
+        "External versioning is not supported by the update action. See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update.html."
+      ) if params['action'] == 'update' and external_version_types.include?(params.fetch('version_type', ''))
+
       # Update API setup
       update_options = {
         :doc_as_upsert => params["doc_as_upsert"],

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '6.1.0'
+  s.version         = '6.2.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'ftw', '~> 0.0.42'
   s.add_development_dependency 'addressable', "~> 2.3.0" # used by FTW. V 2.5.0 is ruby 2.0 only.
   s.add_development_dependency 'logstash-codec-plain'
+  s.add_development_dependency 'json' # used by spec/unit/outputs/elasticsearch/http_client/pool_spec.rb
 
   if RUBY_PLATFORM == 'java'
     s.platform = RUBY_PLATFORM

data/spec/integration/outputs/create_spec.rb

@@ -3,7 +3,7 @@ require_relative "../../../spec/es_spec_helper"
 describe "client create actions", :integration => true do
   require "logstash/outputs/elasticsearch"
 
-  def get_es_output(action, id)
+  def get_es_output(action, id, version=nil, version_type=nil)
     settings = {
       "manage_template" => true,
       "index" => "logstash-create",
@@ -12,6 +12,8 @@ describe "client create actions", :integration => true do
       "action" => action
     }
     settings['document_id'] = id
+    settings['version'] = version if version
+    settings['version_type'] = version_type if version_type
     LogStash::Outputs::ElasticSearch.new(settings)
   end
 
@@ -36,5 +38,30 @@ describe "client create actions", :integration => true do
         insist { r["hits"]["total"] } == 1
       end
     end
+
+    it "should allow default (internal) version" do
+      subject = get_es_output("create", "id123", 43)
+      subject.register
+    end
+
+    it "should allow internal version" do
+      subject = get_es_output("create", "id123", 43, "internal")
+      subject.register
+    end
+
+    it "should not allow external version" do
+      subject = get_es_output("create", "id123", 43, "external")
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+    it "should not allow external_gt version" do
+      subject = get_es_output("create", "id123", 43, "external_gt")
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+    it "should not allow external_gte version" do
+      subject = get_es_output("create", "id123", 43, "external_gte")
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
   end
 end

data/spec/integration/outputs/delete_spec.rb

@@ -0,0 +1,63 @@
+require_relative "../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch"
+
+
+describe "Versioned delete", :integration => true, :version_greater_than_equal_to_2x => true do
+  require "logstash/outputs/elasticsearch"
+
+  let(:es) { get_client }
+
+  before :each do
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    es.indices.delete(:index => "*") rescue nil
+    es.indices.refresh
+  end
+
+  context "when delete only" do
+    subject { LogStash::Outputs::ElasticSearch.new(settings) }
+
+    before do
+      subject.register
+    end
+
+    let(:settings) do
+      {
+        "manage_template" => true,
+        "index" => "logstash-delete",
+        "template_overwrite" => true,
+        "hosts" => get_host_port(),
+        "document_id" => "%{my_id}",
+        "version" => "%{my_version}",
+        "version_type" => "external",
+        "action" => "%{my_action}"
+      }
+    end
+
+    it "should ignore non-monotonic external version updates" do
+      id = "ev2"
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
+      r = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
+      expect(r['_version']).to eq(99)
+      expect(r['_source']['message']).to eq('foo')
+
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
+      r2 = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
+      expect(r2['_version']).to eq(99)
+      expect(r2['_source']['message']).to eq('foo')
+    end
+
+    it "should commit monotonic external version updates" do
+      id = "ev3"
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
+      r = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
+      expect(r['_version']).to eq(99)
+      expect(r['_source']['message']).to eq('foo')
+
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
+      expect { es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+    end
+  end
+end

data/spec/integration/outputs/index_version_spec.rb

@@ -0,0 +1,101 @@
+require_relative "../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch"
+
+
+describe "Versioned indexing", :integration => true, :version_greater_than_equal_to_2x => true do
+  require "logstash/outputs/elasticsearch"
+
+  let(:es) { get_client }
+
+  before :each do
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    es.indices.delete(:index => "*") rescue nil
+    es.indices.refresh
+  end
+
+  context "when index only" do
+    subject { LogStash::Outputs::ElasticSearch.new(settings) }
+
+    before do
+      subject.register
+    end
+
+    describe "unversioned output" do
+      let(:settings) do
+        {
+          "manage_template" => true,
+          "index" => "logstash-index",
+          "template_overwrite" => true,
+          "hosts" => get_host_port(),
+          "action" => "index",
+          "script_lang" => "groovy",
+          "document_id" => "%{my_id}"
+        }
+      end
+
+      it "should default to ES version" do
+        subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
+        r = es.get(:index => 'logstash-index', :type => 'logs', :id => "123", :refresh => true)
+        expect(r["_version"]).to eq(1)
+        expect(r["_source"]["message"]).to eq('foo')
+        subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
+        r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => "123", :refresh => true)
+        expect(r2["_version"]).to eq(2)
+        expect(r2["_source"]["message"]).to eq('foobar')
+      end  
+    end
+
+    describe "versioned output" do
+      let(:settings) do 
+        {
+          "manage_template" => true,
+          "index" => "logstash-index",
+          "template_overwrite" => true,
+          "hosts" => get_host_port(),
+          "action" => "index",
+          "script_lang" => "groovy",
+          "document_id" => "%{my_id}",
+          "version" => "%{my_version}",
+          "version_type" => "external",
+        }
+      end
+
+      it "should respect the external version" do
+        id = "ev1"
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
+        r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
+        expect(r["_version"]).to eq(99)
+        expect(r["_source"]["message"]).to eq('foo')
+      end
+
+      it "should ignore non-monotonic external version updates" do
+        id = "ev2"
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
+        r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
+        expect(r["_version"]).to eq(99)
+        expect(r["_source"]["message"]).to eq('foo')
+
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
+        r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
+        expect(r2["_version"]).to eq(99)
+        expect(r2["_source"]["message"]).to eq('foo')
+      end
+
+      it "should commit monotonic external version updates" do
+        id = "ev3"
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
+        r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
+        expect(r["_version"]).to eq(99)
+        expect(r["_source"]["message"]).to eq('foo')
+
+        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
+        r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
+        expect(r2["_version"]).to eq(100)
+        expect(r2["_source"]["message"]).to eq('foo')
+      end
+    end
+  end
+end

data/spec/integration/outputs/update_spec.rb

@@ -62,6 +62,32 @@ describe "Update actions", :integration => true, :version_greater_than_equal_to_
       insist { r["_source"]["data"] } == 'updated message here'
       insist { r["_source"]["message"] } == 'foo'
     end
+
+    it "should allow default (internal) version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99" })
+      subject.register
+    end
+
+    it "should allow internal version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
+      subject.register
+    end
+
+    it "should not allow external version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+    it "should not allow external_gt version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+    it "should not allow external_gte version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
   end
 
   context "when using script" do
@@ -154,6 +180,11 @@ describe "Update actions", :integration => true, :version_greater_than_equal_to_
       insist { r["_source"]["message"] } == 'sample message here'
     end
 
+    it "should fail on documents with event/doc as upsert at external version" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
     context "when using script" do
       it "should create new documents with upsert content" do
         subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 6.1.0
+  version: 6.2.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-27 00:00:00.000000000 Z
+date: 2017-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -106,6 +106,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: json
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -196,7 +210,9 @@ files:
 - spec/fixtures/scripts/scripted_update_nested.groovy
 - spec/fixtures/scripts/scripted_upsert.groovy
 - spec/integration/outputs/create_spec.rb
+- spec/integration/outputs/delete_spec.rb
 - spec/integration/outputs/index_spec.rb
+- spec/integration/outputs/index_version_spec.rb
 - spec/integration/outputs/parent_spec.rb
 - spec/integration/outputs/pipeline_spec.rb
 - spec/integration/outputs/retry_spec.rb
@@ -247,7 +263,9 @@ test_files:
 - spec/fixtures/scripts/scripted_update_nested.groovy
 - spec/fixtures/scripts/scripted_upsert.groovy
 - spec/integration/outputs/create_spec.rb
+- spec/integration/outputs/delete_spec.rb
 - spec/integration/outputs/index_spec.rb
+- spec/integration/outputs/index_version_spec.rb
 - spec/integration/outputs/parent_spec.rb
 - spec/integration/outputs/pipeline_spec.rb
 - spec/integration/outputs/retry_spec.rb