logstash-output-amazon_es 2.0.1-java → 6.4.0-java

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -0,0 +1,60 @@
+require "logstash/outputs/amazon_es"
+require_relative "../../../spec/es_spec_helper"
+
+describe "whitelisting error types in expected behavior" do
+  let(:template) { '{"template" : "not important, will be updated by :index"}' }
+  let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z") }
+  let(:action1) { ["index", {:_id=>1, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"doc"}, event1] }
+  let(:settings) { {"manage_template" => true,
+                    "index" => "logstash-2014.11.17",
+                    "template_overwrite" => true,
+                    "hosts" => get_host_port(),
+                    "protocol" => "http",
+                    "port" => 9200,
+                    "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
+                    "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
+
+  subject { LogStash::Outputs::ElasticSearch.new(settings) }
+
+  before :each do
+    allow(subject.logger).to receive(:warn)
+
+    subject.register
+
+    allow(subject.client).to receive(:maximum_seen_major_version).and_return(0)
+    allow(subject.client).to receive(:bulk).and_return(
+      {
+        "errors" => true,
+        "items" => [{
+          "create" => {
+            "status" => 409, 
+            "error" => {
+              "type" => "document_already_exists_exception",
+              "reason" => "[shard] document already exists"
+            }
+          }
+        }]
+      })
+
+    subject.multi_receive([event1])
+  end
+
+  after :each do
+    subject.close
+  end
+
+  describe "when failure logging is enabled for everything" do
+    it "should log a failure on the action" do
+      expect(subject.logger).to have_received(:warn).with("Failed action.", anything)
+    end
+  end
+
+  describe "when failure logging is disabled for docuemnt exists error" do
+    let(:settings) { super.merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) }
+
+    it "should log a failure on the action" do
+      expect(subject.logger).not_to have_received(:warn).with("Failed action.", anything)
+    end
+  end
+
+end

metadata

@@ -1,49 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-amazon_es
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 6.4.0
 platform: java
 authors:
 - Amazon
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-30 00:00:00.000000000 Z
+date: 2018-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  name: concurrent-ruby
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.0.10
+        version: 0.5.4
     - - "<"
       - !ruby/object:Gem::Version
-        version: 6.0.0
-  name: elasticsearch
+        version: 1.0.0
+  name: manticore
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.10
+        version: 0.5.4
     - - "<"
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -123,117 +109,55 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.1
-  name: faraday
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.9.1
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.10.0
-  name: faraday_middleware
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.10.0
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 2.5.0
-  name: addressable
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 2.5.0
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.0.42
-  name: ftw
+        version: '0'
+  name: logstash-codec-plain
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.42
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-input-generator
+  name: logstash-devutils
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.5.2
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-  name: manticore
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.5.2
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
+  name: flores
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
-  name: longshoreman
+  name: elasticsearch
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 description: Output events to Amazon Elasticsearch Service with V4 signing
@@ -242,24 +166,35 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
 - README.md
 - lib/logstash/outputs/amazon_es.rb
-- lib/logstash/outputs/amazon_es/aws_transport.rb
-- lib/logstash/outputs/amazon_es/aws_v4_signer.rb
-- lib/logstash/outputs/amazon_es/aws_v4_signer_impl.rb
-- lib/logstash/outputs/amazon_es/elasticsearch-template.json
+- lib/logstash/outputs/amazon_es/common.rb
+- lib/logstash/outputs/amazon_es/common_configs.rb
+- lib/logstash/outputs/amazon_es/elasticsearch-template-es2x.json
+- lib/logstash/outputs/amazon_es/elasticsearch-template-es5x.json
+- lib/logstash/outputs/amazon_es/elasticsearch-template-es6x.json
+- lib/logstash/outputs/amazon_es/elasticsearch-template-es7x.json
 - lib/logstash/outputs/amazon_es/http_client.rb
+- lib/logstash/outputs/amazon_es/http_client/manticore_adapter.rb
+- lib/logstash/outputs/amazon_es/http_client/pool.rb
+- lib/logstash/outputs/amazon_es/http_client_builder.rb
+- lib/logstash/outputs/amazon_es/template_manager.rb
 - logstash-output-amazon_es.gemspec
-- spec/amazon_es_spec_helper.rb
-- spec/unit/outputs/amazon_es_spec.rb
-- spec/unit/outputs/elasticsearch/protocol_spec.rb
-- spec/unit/outputs/elasticsearch_proxy_spec.rb
-homepage: http://logstash.net/
+- spec/es_spec_helper.rb
+- spec/unit/http_client_builder_spec.rb
+- spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb
+- spec/unit/outputs/elasticsearch/http_client/pool_spec.rb
+- spec/unit/outputs/elasticsearch/http_client_spec.rb
+- spec/unit/outputs/elasticsearch/template_manager_spec.rb
+- spec/unit/outputs/elasticsearch_spec.rb
+- spec/unit/outputs/error_whitelist_spec.rb
+homepage: https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/index.html
 licenses:
-- apache-2.0
+- Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
@@ -279,12 +214,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.8
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Logstash Output to Amazon Elasticsearch Service
 test_files:
-- spec/amazon_es_spec_helper.rb
-- spec/unit/outputs/amazon_es_spec.rb
-- spec/unit/outputs/elasticsearch/protocol_spec.rb
-- spec/unit/outputs/elasticsearch_proxy_spec.rb
+- spec/es_spec_helper.rb
+- spec/unit/http_client_builder_spec.rb
+- spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb
+- spec/unit/outputs/elasticsearch/http_client/pool_spec.rb
+- spec/unit/outputs/elasticsearch/http_client_spec.rb
+- spec/unit/outputs/elasticsearch/template_manager_spec.rb
+- spec/unit/outputs/elasticsearch_spec.rb
+- spec/unit/outputs/error_whitelist_spec.rb

data/lib/logstash/outputs/amazon_es/aws_transport.rb

@@ -1,109 +0,0 @@
-require 'faraday'
-require 'faraday_middleware'
-require 'aws-sdk-core'
-require 'elasticsearch'
-require 'elasticsearch-transport'
-require 'manticore'
-require 'faraday/adapter/manticore'
-
-#
-require 'uri'
-
-require_relative "aws_v4_signer"
-
-
-module Elasticsearch
-  module Transport
-    module Transport
-      module HTTP
-
-        # Transport implementation, which V4 Signs requests using the [_Faraday_](https://rubygems.org/gems/faraday)
-        # library for abstracting the HTTP client.
-        #
-        # @see Transport::Base
-        #
-        class AWS
-          include Elasticsearch::Transport::Transport::Base
-          
-
-          DEFAULT_PORT = 80
-          DEFAULT_PROTOCOL = "http"    
-          
-          CredentialConfig = Struct.new(
-            :access_key_id,
-            :secret_access_key,
-            :session_token,
-            :profile,
-            :instance_profile_credentials_retries,
-            :instance_profile_credentials_timeout,
-            :region
-          )      
-
-          # Performs the request by invoking {Transport::Base#perform_request} with a block.
-          #
-          # @return [Response]
-          # @see    Transport::Base#perform_request
-          #
-          def perform_request(method, path, params={}, body=nil)
-            super do |connection, url|
-              response = connection.connection.run_request \
-               method.downcase.to_sym,
-                url,
-                ( body ? __convert_to_json(body) : nil ),
-                {}
-
-              Response.new response.status, response.body, response.headers
-            end
-          end
-
-          # Builds and returns a collection of connections.
-          #
-          # @return [Connections::Collection]
-          #
-          def __build_connections
-            region = options[:region]
-            access_key_id = options[:aws_access_key_id] || nil
-            secret_access_key = options[:aws_secret_access_key] || nil
-            session_token = options[:session_token] || nil
-            profile = options[:profile] || 'default'
-            instance_cred_retries = options[:instance_profile_credentials_retries] || 0
-            instance_cred_timeout = options[:instance_profile_credentials_timeout] || 1
-            
-            credential_config = CredentialConfig.new(access_key_id, secret_access_key, session_token, profile,
-                                                     instance_cred_retries, instance_cred_timeout, region)
-            credentials = Aws::CredentialProviderChain.new(credential_config).resolve
-
-            Connections::Collection.new \
-              :connections => hosts.map { |host|
-                host[:protocol]   = host[:scheme] || DEFAULT_PROTOCOL
-                host[:port]     ||= DEFAULT_PORT
-                url               = __full_url(host)
-                                  
-                aes_connection = ::Faraday::Connection.new(url,  (options[:transport_options] || {})) do |faraday|
-                  faraday.request :aws_v4_signer,
-                                        credentials: credentials,
-                                        service_name: 'es',
-                                        region: region
-                  faraday.adapter :manticore
-                end
-                
-                Connections::Connection.new \
-                  :host => host,
-                  :connection => aes_connection
-              },
-              :selector_class => options[:selector_class],
-              :selector => options[:selector]
-          end
-
-          # Returns an array of implementation specific connection errors.
-          #
-          # @return [Array]
-          #
-          def host_unreachable_exceptions
-            [::Faraday::Error::ConnectionFailed, ::Faraday::Error::TimeoutError]
-          end
-        end
-      end
-    end
-  end
-end

data/lib/logstash/outputs/amazon_es/aws_v4_signer.rb

@@ -1,7 +0,0 @@
-require 'faraday'
-require_relative 'aws_v4_signer_impl'
-
-module FaradayMiddleware
-  Faraday::Request.register_middleware :aws_v4_signer => lambda { AwsV4Signer }
-end
-

data/lib/logstash/outputs/amazon_es/aws_v4_signer_impl.rb

@@ -1,62 +0,0 @@
-require 'aws-sdk-core'
-require 'faraday'
-
-
-class AwsV4Signer < Faraday::Middleware
-  class Request
-    def initialize(env)
-      @env = env
-    end
-
-    def headers
-      @env.request_headers
-    end
-
-    def set_header(header)
-      @env.request_headers = header
-    end
-
-    def body
-      @env.body || ''
-    end
-
-    def endpoint
-      @env.url
-    end
-
-    def http_method
-      @env.method.to_s.upcase
-    end
-  end
-
-  def signer
-    credentials = @options.fetch(:credentials)
-    service_name = @options.fetch(:service_name)
-    region = @options.fetch(:region)
-    Aws::Signers::V4.new(credentials, service_name, region)
-  end
-
- def initialize(app, options = nil)
-    super(app)
-    @options = options
-    @net_http = app.is_a?(Faraday::Adapter::NetHttp)
-  end
-
-  def call(env)
-    normalize_for_net_http!(env)
-    req = Request.new(env)
-    signer().sign(req)
-    @app.call(env)
-  end
-
-  private
-  def normalize_for_net_http!(env)
-    return unless @net_http
-
-    if Net::HTTP::HAVE_ZLIB
-      env.request_headers['Accept-Encoding'] ||= 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3'
-    end
-
-    env.request_headers['Accept'] ||= '*/*'
-  end
-end