logstash-output-amazon_es 2.0.1-java → 6.4.0-java

data/lib/logstash/outputs/amazon_es/http_client_builder.rb

@@ -0,0 +1,164 @@
+require 'cgi'
+
+module LogStash; module Outputs; class ElasticSearch;
+  module HttpClientBuilder
+    def self.build(logger, hosts, params)
+      client_settings = {
+        :pool_max => params["pool_max"],
+        :pool_max_per_route => params["pool_max_per_route"],
+        :check_connection_timeout => params["validate_after_inactivity"],
+        :http_compression => params["http_compression"],
+        :headers => params["custom_headers"]
+      }
+      
+      client_settings[:proxy] = params["proxy"] if params["proxy"]
+      
+      common_options = {
+        :client_settings => client_settings,
+        :metric => params["metric"],
+        :resurrect_delay => params["resurrect_delay"]
+      }
+
+      if params["sniffing"]
+        common_options[:sniffing] = true
+        common_options[:sniffer_delay] = params["sniffing_delay"]
+      end
+
+      common_options[:timeout] = params["timeout"] if params["timeout"]
+
+      if params["path"]
+        client_settings[:path] = dedup_slashes("/#{params["path"]}/")
+      end
+
+      common_options[:bulk_path] = if params["bulk_path"]
+         dedup_slashes("/#{params["bulk_path"]}")
+      else
+         dedup_slashes("/#{params["path"]}/_bulk")
+      end
+
+      common_options[:sniffing_path] = if params["sniffing_path"]
+         dedup_slashes("/#{params["sniffing_path"]}")
+      else
+         dedup_slashes("/#{params["path"]}/_nodes/http")
+      end
+
+      common_options[:healthcheck_path] = if params["healthcheck_path"]
+         dedup_slashes("/#{params["healthcheck_path"]}")
+      else
+         dedup_slashes("/#{params["path"]}")
+      end
+
+      if params["parameters"]
+        client_settings[:parameters] = params["parameters"]
+      end
+
+      logger.debug? && logger.debug("Normalizing http path", :path => params["path"], :normalized => client_settings[:path])
+
+      client_settings.merge! setup_ssl(logger, params)
+      common_options.merge! setup_basic_auth(logger, params)
+
+      external_version_types = ["external", "external_gt", "external_gte"]
+      # External Version validation
+      raise(
+        LogStash::ConfigurationError,
+        "External versioning requires the presence of a version number."
+      ) if external_version_types.include?(params.fetch('version_type', '')) and params.fetch("version", nil) == nil
+ 
+
+      # Create API setup
+      raise(
+        LogStash::ConfigurationError,
+        "External versioning is not supported by the create action."
+      ) if params['action'] == 'create' and external_version_types.include?(params.fetch('version_type', ''))
+
+      # Update API setup
+      raise( LogStash::ConfigurationError,
+        "doc_as_upsert and scripted_upsert are mutually exclusive."
+      ) if params["doc_as_upsert"] and params["scripted_upsert"]
+
+      raise(
+        LogStash::ConfigurationError,
+        "Specifying action => 'update' needs a document_id."
+      ) if params['action'] == 'update' and params.fetch('document_id', '') == ''
+
+      raise(
+        LogStash::ConfigurationError,
+        "External versioning is not supported by the update action. See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update.html."
+      ) if params['action'] == 'update' and external_version_types.include?(params.fetch('version_type', ''))
+
+      # Update API setup
+      update_options = {
+        :doc_as_upsert => params["doc_as_upsert"],
+        :script_var_name => params["script_var_name"],
+        :script_type => params["script_type"],
+        :script_lang => params["script_lang"],
+        :scripted_upsert => params["scripted_upsert"]
+      }
+      common_options.merge! update_options if params["action"] == 'update'
+      create_http_client(common_options.merge(:hosts => hosts,
+                                              :logger => logger,
+                                              :protocol => params["protocol"],
+                                              :port => params["port"],
+                                              :region => params["region"],
+                                              :aws_access_key_id => params["aws_access_key_id"],
+                                              :aws_secret_access_key => params["aws_secret_access_key"]))
+    end
+
+    def self.create_http_client(options)
+      LogStash::Outputs::ElasticSearch::HttpClient.new(options)
+    end
+
+    def self.setup_ssl(logger, params)
+      params["ssl"] = true if params["hosts"].any? {|h| h.scheme == "https" }
+      return {} if params["ssl"].nil?
+
+      return {:ssl => {:enabled => false}} if params["ssl"] == false
+
+      cacert, truststore, truststore_password, keystore, keystore_password =
+        params.values_at('cacert', 'truststore', 'truststore_password', 'keystore', 'keystore_password')
+
+      if cacert && truststore
+        raise(LogStash::ConfigurationError, "Use either \"cacert\" or \"truststore\" when configuring the CA certificate") if truststore
+      end
+
+      ssl_options = {:enabled => true}
+
+      if cacert
+        ssl_options[:ca_file] = cacert
+      elsif truststore
+        ssl_options[:truststore_password] = truststore_password.value if truststore_password
+      end
+
+      ssl_options[:truststore] = truststore if truststore
+      if keystore
+        ssl_options[:keystore] = keystore
+        ssl_options[:keystore_password] = keystore_password.value if keystore_password
+      end
+      if !params["ssl_certificate_verification"]
+        logger.warn [
+                       "** WARNING ** Detected UNSAFE options in amazon_es output configuration!",
+                       "** WARNING ** You have enabled encryption but DISABLED certificate verification.",
+                       "** WARNING ** To make sure your data is secure change :ssl_certificate_verification to true"
+                     ].join("\n")
+        ssl_options[:verify] = false
+      end
+      { ssl: ssl_options }
+    end
+
+    def self.setup_basic_auth(logger, params)
+      user, password = params["user"], params["password"]
+      
+      return {} unless user && password && password.value
+
+      {
+        :user => CGI.escape(user),
+        :password => CGI.escape(password.value)
+      }
+    end
+
+    private
+    def self.dedup_slashes(url)
+      url.gsub(/\/+/, "/")
+    end
+  end
+end; end; end

data/lib/logstash/outputs/amazon_es/template_manager.rb

@@ -0,0 +1,36 @@
+module LogStash; module Outputs; class ElasticSearch
+  class TemplateManager
+    # To be mixed into the amazon_es plugin base
+    def self.install_template(plugin)
+      return unless plugin.manage_template
+      plugin.logger.info("Using mapping template from", :path => plugin.template)
+      template = get_template(plugin.template, plugin.maximum_seen_major_version)
+      plugin.logger.info("Attempting to install template", :manage_template => template)
+      install(plugin.client, plugin.template_name, template, plugin.template_overwrite)
+    rescue => e
+      plugin.logger.error("Failed to install template.", :message => e.message, :class => e.class.name, :backtrace => e.backtrace)
+    end
+
+    private
+    def self.get_template(path, es_major_version)
+      template_path = path || default_template_path(es_major_version)
+      read_template_file(template_path)
+    end
+
+    def self.install(client, template_name, template, template_overwrite)
+      client.template_install(template_name, template, template_overwrite)
+    end
+
+    def self.default_template_path(es_major_version)
+      template_version = es_major_version == 1 ? 2 : es_major_version
+      default_template_name = "elasticsearch-template-es#{template_version}x.json"
+      ::File.expand_path(default_template_name, ::File.dirname(__FILE__))
+    end
+
+    def self.read_template_file(template_path)
+      raise ArgumentError, "Template file '#{@template_path}' could not be found!" unless ::File.exists?(template_path)
+      template_data = ::IO.read(template_path)
+      LogStash::Json.load(template_data)
+    end
+  end
+end end end

data/logstash-output-amazon_es.gemspec

@@ -1,17 +1,18 @@
 Gem::Specification.new do |s|
-
   s.name            = 'logstash-output-amazon_es'
-  s.version         = '2.0.1'
-  s.licenses        = ['apache-2.0']
+  s.version         = '6.4.0'
+  s.licenses        = ['Apache-2.0']
   s.summary         = "Logstash Output to Amazon Elasticsearch Service"
   s.description     = "Output events to Amazon Elasticsearch Service with V4 signing"
   s.authors         = ["Amazon"]
   s.email           = 'feedback-prod-elasticsearch@amazon.com'
-  s.homepage        = "http://logstash.net/"
+  s.homepage        = "https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/index.html"
   s.require_paths = ["lib"]
 
+  s.platform = RUBY_PLATFORM
+
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -19,25 +20,15 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
-  # Gem dependencies
-  s.add_runtime_dependency 'concurrent-ruby'
-  s.add_runtime_dependency 'elasticsearch', '>= 1.0.10', '< 6.0.0'
+  s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency 'cabin', ['~> 0.6']
-  s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'aws-sdk', '>= 2.3.22', '~> 2'
-  s.add_runtime_dependency "faraday", '~> 0.9.1'
-  s.add_runtime_dependency "faraday_middleware", '~> 0.10.0'
-
-  s.add_development_dependency 'addressable', '< 2.5.0'
-  s.add_development_dependency 'ftw', '~> 0.0.42'
-  s.add_development_dependency 'logstash-input-generator'
-
-  if RUBY_PLATFORM == 'java'
-    s.platform = RUBY_PLATFORM
-    s.add_runtime_dependency "manticore", '>= 0.5.2', '< 1.0.0'
-  end
 
-  s.add_development_dependency 'logstash-devutils'
-  s.add_development_dependency 'longshoreman'
+  s.add_development_dependency 'logstash-codec-plain', '~> 0'
+  s.add_development_dependency 'logstash-devutils', '~> 0'
+  s.add_development_dependency 'flores', '~> 0'
+  # Still used in some specs, we should remove this ASAP
+  s.add_development_dependency 'elasticsearch', '~> 0'
 end

data/spec/es_spec_helper.rb

@@ -0,0 +1,37 @@
+require "logstash/devutils/rspec/spec_helper"
+require 'manticore'
+require 'elasticsearch'
+
+# by default exclude secure_integration tests unless requested
+# normal integration specs are already excluded by devutils' spec helper
+RSpec.configure do |config|
+   config.filter_run_excluding config.exclusion_filter.add(:secure_integration => true)
+end
+
+module ESHelper
+  def get_host_port
+    "127.0.0.1"
+  end
+
+  def get_client
+    Elasticsearch::Client.new(:hosts => [get_host_port])
+  end
+
+  def self.es_version
+    RSpec.configuration.filter[:es_version] || ENV['ES_VERSION']
+  end
+
+  def self.es_version_satisfies?(*requirement)
+    es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION']
+    if es_version.nil?
+      puts "Info: ES_VERSION environment or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call."
+      return false
+    end
+    es_release_version = Gem::Version.new(es_version).release
+    Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
+  end
+end
+
+RSpec.configure do |config|
+  config.include ESHelper
+end

data/spec/unit/http_client_builder_spec.rb

@@ -0,0 +1,189 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/amazon_es"
+require "logstash/outputs/amazon_es/http_client"
+require "logstash/outputs/amazon_es/http_client_builder"
+
+describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
+  describe "auth setup with url encodable passwords" do
+    let(:klass) { LogStash::Outputs::ElasticSearch::HttpClientBuilder }
+    let(:user) { "foo@bar"}
+    let(:password) {"baz@blah" }
+    let(:password_secured) do
+      secured = double("password")
+      allow(secured).to receive(:value).and_return(password)
+      secured
+    end
+    let(:options) { {"user" => user, "password" => password} }
+    let(:logger) { mock("logger") }
+    let(:auth_setup) { klass.setup_basic_auth(double("logger"), {"user" => user, "password" => password_secured}) }
+
+    it "should return the user escaped" do
+      expect(auth_setup[:user]).to eql(CGI.escape(user))
+    end
+
+    it "should return the password escaped" do
+      expect(auth_setup[:password]).to eql(CGI.escape(password))
+    end
+  end
+
+  describe "customizing action paths" do
+    let(:hosts) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200") ] }
+    let(:options) { {"hosts" => hosts ,
+                     "protocol" => "http",
+                     "port" => 9200,
+                     "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
+                     "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
+    let(:logger) { double("logger") }
+    before :each do
+      [:debug, :debug?, :info?, :info, :warn].each do |level|
+        allow(logger).to receive(level)
+      end
+    end
+
+    describe "healthcheck_path" do
+
+      context "when setting bulk_path" do
+        let(:bulk_path) { "/meh" }
+        let(:options) { super.merge("bulk_path" => bulk_path) }
+
+        context "when using path" do
+          let(:options) { super.merge("path" => "/path") }
+          it "ignores the path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq(bulk_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+        context "when not using path" do
+
+          it "uses the bulk_path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq(bulk_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+
+      context "when not setting bulk_path" do
+
+        context "when using path" do
+          let(:path) { "/meh" }
+          let(:options) { super.merge("path" => path) }
+          it "sets bulk_path to path+_bulk" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq("#{path}/_bulk")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+
+        context "when not using path" do
+          it "sets the bulk_path to _bulk" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq("/_bulk")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+    end
+    describe "healthcheck_path" do
+      context "when setting healthcheck_path" do
+        let(:healthcheck_path) { "/meh" }
+        let(:options) { super.merge("healthcheck_path" => healthcheck_path) }
+
+        context "when using path" do
+          let(:options) { super.merge("path" => "/path") }
+          it "ignores the path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq(healthcheck_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+        context "when not using path" do
+
+          it "uses the healthcheck_path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq(healthcheck_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+
+      context "when not setting healthcheck_path" do
+
+        context "when using path" do
+          let(:path) { "/meh" }
+          let(:options) { super.merge("path" => path) }
+          it "sets healthcheck_path to path" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq(path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+
+        context "when not using path" do
+          it "sets the healthcheck_path to root" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq("/")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+    end
+    describe "sniffing_path" do
+      context "when setting sniffing_path" do
+        let(:sniffing_path) { "/meh" }
+        let(:options) { super.merge("sniffing_path" => sniffing_path) }
+
+        context "when using path" do
+          let(:options) { super.merge("path" => "/path") }
+          it "ignores the path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq(sniffing_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+        context "when not using path" do
+
+          it "uses the sniffing_path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq(sniffing_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+
+      context "when not setting sniffing_path" do
+
+        context "when using path" do
+          let(:path) { "/meh" }
+          let(:options) { super.merge("path" => path) }
+          it "sets sniffing_path to path+_nodes/http" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq("#{path}/_nodes/http")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+
+        context "when not using path" do
+          it "sets the sniffing_path to _nodes/http" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq("/_nodes/http")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+    end
+  end
+end