logstash-output-elasticsearch_java 1.0.0.beta1

data/spec/integration/outputs/update_spec.rb

@@ -0,0 +1,88 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "all protocols update actions", :integration => true do
+  require "logstash/outputs/elasticsearch_java"
+  require "elasticsearch"
+
+  def get_es_output( protocol, id = nil, upsert = nil, doc_as_upsert=nil)
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-update",
+      "template_overwrite" => true,
+      "protocol" => protocol,
+      "hosts" => get_host(),
+      "port" => get_port(protocol),
+      "action" => "update"
+    }
+    settings['upsert'] = upsert unless upsert.nil?
+    settings['document_id'] = id unless id.nil?
+    settings['doc_as_upsert'] = doc_as_upsert unless doc_as_upsert.nil?
+
+    LogStash::Outputs::ElasticSearchJava.new(settings)
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    @es.index(
+      :index => 'logstash-update',
+      :type => 'logs',
+      :id => "123",
+      :body => { :message => 'Test' }
+    )
+    @es.indices.refresh
+  end
+
+  ["node", "transport"].each do |protocol|
+    context "update only with #{protocol} protocol" do
+      it "should failed without a document_id" do
+        event = LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0})
+        action = ["update", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event]
+        subject = get_es_output(protocol)
+        subject.register
+        expect { subject.flush([action]) }.to raise_error
+      end
+
+      it "should not create new document" do
+        subject = get_es_output(protocol, "456")
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.buffer_flush(:final => true)
+        expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      end
+
+      it "should update existing document" do
+        subject = get_es_output(protocol, "123")
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "updated message here"))
+        subject.buffer_flush(:final => true)
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        insist { r["_source"]["message"] } == 'updated message here'
+      end
+    end
+
+    context "upsert with #{protocol} protocol" do
+      it "should create new documents with upsert content" do
+        subject = get_es_output(protocol, "456", '{"message": "upsert message"}')
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.buffer_flush(:final => true)
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["message"] } == 'upsert message'
+      end
+
+      it "should create new documents with event/doc as upsert" do
+        subject = get_es_output(protocol, "456", nil, true)
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.buffer_flush(:final => true)
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["message"] } == 'sample message here'
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/protocol_spec.rb

@@ -0,0 +1,32 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch_java/protocol"
+require "java"
+
+describe LogStash::Outputs::ElasticSearchJavaPlugins::Protocols::NodeClient do
+  context "successful" do
+    it "should map correctly" do
+      index_response = org.elasticsearch.action.index.IndexResponse.new("my_index", "my_type", "my_id", 123, true)
+      update_response = org.elasticsearch.action.update.UpdateResponse.new("my_index", "my_type", "my_id", 123, false)
+      delete_response = org.elasticsearch.action.delete.DeleteResponse.new("my_index", "my_type", "my_id", 123, true)
+      bulk_item_response_index = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "index", index_response)
+      bulk_item_response_update = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "update", update_response)
+      bulk_item_response_delete = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "delete", delete_response)
+      bulk_response = org.elasticsearch.action.bulk.BulkResponse.new([bulk_item_response_index, bulk_item_response_update, bulk_item_response_delete], 0)
+      ret = LogStash::Outputs::ElasticSearchJavaPlugins::Protocols::NodeClient.normalize_bulk_response(bulk_response)
+      insist { ret } == {"errors" => false}
+    end
+  end
+
+  context "contains failures" do
+    it "should map correctly" do
+      failure = org.elasticsearch.action.bulk.BulkItemResponse::Failure.new("my_index", "my_type", "my_id", java.lang.IllegalArgumentException.new("bad_request"))
+      failure_code = failure.get_status.status
+      bulk_item_response_index = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "index", failure)
+      bulk_item_response_update = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "update", failure)
+      bulk_item_response_delete = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "delete", failure)
+      bulk_response = org.elasticsearch.action.bulk.BulkResponse.new([bulk_item_response_index, bulk_item_response_update, bulk_item_response_delete], 0)
+      actual = LogStash::Outputs::ElasticSearchJavaPlugins::Protocols::NodeClient.normalize_bulk_response(bulk_response)
+      insist { actual } == {"errors" => true, "statuses" => [failure_code, failure_code, failure_code]}
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -0,0 +1,79 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "outputs/elasticsearch_java" do
+  context "registration" do
+    it "should register" do
+      output = LogStash::Plugin.lookup("output", "elasticsearch_java").new("protocol" => "transport", "manage_template" => "false")
+      # register will try to load jars and raise if it cannot find jars
+      expect {output.register}.to_not raise_error
+    end
+  end
+
+  describe "transport protocol" do
+    context "host not configured" do
+      subject do
+        require "logstash/outputs/elasticsearch_java"
+        settings = {
+          "protocol" => "transport",
+          "node_name" => "mynode"
+        }
+        next LogStash::Outputs::ElasticSearchJava.new(settings)
+      end
+
+      it "should set host to localhost" do
+        expect(LogStash::Outputs::ElasticSearchJavaPlugins::Protocols::TransportClient).to receive(:new).with({
+          :hosts => ["127.0.0.1"],
+          :port => "9300-9305",
+          :protocol => "transport",
+          :client_settings => {
+            "client.transport.sniff" => false,
+            "node.name" => "mynode"
+          }
+        })
+        subject.register
+      end
+    end
+
+    context "sniffing => true" do
+      subject do
+        require "logstash/outputs/elasticsearch_java"
+        settings = {
+          "hosts" => "node01",
+          "protocol" => "transport",
+          "sniffing" => true
+        }
+        next LogStash::Outputs::ElasticSearchJava.new(settings)
+      end
+
+      it "should set the sniffing property to true" do
+        allow_any_instance_of(LogStash::Outputs::ElasticSearchJavaPlugins::Protocols::NodeClient).to receive(:client).and_return(nil)
+        subject.register
+        client = subject.instance_eval("@client")
+        settings = client.instance_eval("@settings")
+
+        expect(settings.build.get("client.transport.sniff")).to eq("true")
+      end
+    end
+
+    context "sniffing => false" do
+      subject do
+        require "logstash/outputs/elasticsearch_java"
+        settings = {
+          "hosts" => "node01",
+          "protocol" => "transport",
+          "sniffing" => false
+        }
+        next LogStash::Outputs::ElasticSearchJava.new(settings)
+      end
+
+      it "should set the sniffing property to true" do
+        allow_any_instance_of(LogStash::Outputs::ElasticSearchJavaPlugins::Protocols::NodeClient).to receive(:client).and_return(nil)
+        subject.register
+        client = subject.instance_eval("@client")
+        settings = client.instance_eval("@settings")
+
+        expect(settings.build.getAsMap["client.transport.sniff"]).to eq("false")
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,241 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-elasticsearch_java
+version: !ruby/object:Gem::Version
+  version: 1.0.0.beta1
+platform: ruby
+authors:
+- Elastic
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-09-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: concurrent-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.10
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: elasticsearch
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.10
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.17
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  name: stud
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.17
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  name: cabin
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.42
+  name: ftw
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.42
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-input-generator
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: longshoreman
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Output events to elasticsearch using the java client
+email: info@elastic.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- Rakefile
+- lib/logstash-output-elasticsearch_java_jars.rb
+- lib/logstash/outputs/elasticsearch_java.rb
+- lib/logstash/outputs/elasticsearch_java/elasticsearch-template.json
+- lib/logstash/outputs/elasticsearch_java/protocol.rb
+- logstash-output-elasticsearch_java.gemspec
+- spec/es_spec_helper.rb
+- spec/integration/outputs/elasticsearch/node_spec.rb
+- spec/integration/outputs/index_spec.rb
+- spec/integration/outputs/retry_spec.rb
+- spec/integration/outputs/routing_spec.rb
+- spec/integration/outputs/secure_spec.rb
+- spec/integration/outputs/templates_spec.rb
+- spec/integration/outputs/transport_create_spec.rb
+- spec/integration/outputs/update_spec.rb
+- spec/unit/outputs/elasticsearch/protocol_spec.rb
+- spec/unit/outputs/elasticsearch_spec.rb
+- vendor/jar-dependencies/runtime-jars/antlr-runtime-3.5.jar
+- vendor/jar-dependencies/runtime-jars/asm-4.1.jar
+- vendor/jar-dependencies/runtime-jars/asm-commons-4.1.jar
+- vendor/jar-dependencies/runtime-jars/elasticsearch-1.7.0.jar
+- vendor/jar-dependencies/runtime-jars/lucene-analyzers-common-4.10.4.jar
+- vendor/jar-dependencies/runtime-jars/lucene-core-4.10.4.jar
+- vendor/jar-dependencies/runtime-jars/lucene-grouping-4.10.4.jar
+- vendor/jar-dependencies/runtime-jars/lucene-highlighter-4.10.4.jar
+- vendor/jar-dependencies/runtime-jars/lucene-join-4.10.4.jar
+- vendor/jar-dependencies/runtime-jars/lucene-memory-4.10.4.jar
+- vendor/jar-dependencies/runtime-jars/lucene-misc-4.10.4.jar
+- vendor/jar-dependencies/runtime-jars/lucene-queries-4.10.4.jar
+- vendor/jar-dependencies/runtime-jars/lucene-queryparser-4.10.4.jar
+- vendor/jar-dependencies/runtime-jars/lucene-sandbox-4.10.4.jar
+- vendor/jar-dependencies/runtime-jars/lucene-spatial-4.10.4.jar
+- vendor/jar-dependencies/runtime-jars/lucene-suggest-4.10.4.jar
+- vendor/jar-dependencies/runtime-jars/spatial4j-0.4.1.jar
+homepage: http://logstash.net/
+licenses:
+- apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: Logstash Output to Elasticsearch using Java node/transport client
+test_files:
+- spec/es_spec_helper.rb
+- spec/integration/outputs/elasticsearch/node_spec.rb
+- spec/integration/outputs/index_spec.rb
+- spec/integration/outputs/retry_spec.rb
+- spec/integration/outputs/routing_spec.rb
+- spec/integration/outputs/secure_spec.rb
+- spec/integration/outputs/templates_spec.rb
+- spec/integration/outputs/transport_create_spec.rb
+- spec/integration/outputs/update_spec.rb
+- spec/unit/outputs/elasticsearch/protocol_spec.rb
+- spec/unit/outputs/elasticsearch_spec.rb