logstash-output-elasticsearch 0.2.7-java → 0.2.8-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f1be8f824568cc3854051fe53682413cdcf9fd4
-  data.tar.gz: 292fccfa2f2bce0dd60fd6b8218ad5e296667272
+  metadata.gz: d77d5d461770a2ff1f5305fc51bfd2d26ee30765
+  data.tar.gz: 5f7fc066a058c2858684080f7ff428019e44541d
 SHA512:
-  metadata.gz: c48395f88ebf9f24563f62ec72146015a91c55244b0c963c063ebc8717fa7beedd7cd4d56486245a57de55e68d8a5cd785a2d30370768c8eaa75cc0c7911a0a2
-  data.tar.gz: 507f08e66490334416bfc6f64bb5364cf0dd1126ebf7496ab45f98cd750850cd86c858a2a00d4c57951c97132975d0c5a202576e1f48e55febe39185a7b49bfd
+  metadata.gz: bb51c75b6dabfe4dd33f96322ebe920cb64bfb580aaed2215ecc8bb36ae0fb3f58533ef501dbe93b9c56450b7a54333c56ffd72f31b8206574656b5cc5654b6a
+  data.tar.gz: aaf694ad4c3113b7d25a2320d05c66bf4906e5714901f6ba4f6cb2343f302561375128a0002d5eda6c31bceb0b5ea0d984cc1a0c6bd46c43fd623d38e57e4ff1

data/CHANGELOG.md

@@ -0,0 +1,9 @@
+## 0.2.8 (June 12, 2015)
+ - Add option to enable and disable SSL certificate verification during handshake (#160)
+ - Doc improvements for clarifying round robin behavior using hosts config
+
+## 0.2.7 (May 28, 2015)
+ - Bump es-ruby version to 1.0.10
+
+## 0.2.6 (May 28, 2015)
+ - Disable timeouts when using http protocol which would cause bulk requests to fail (#103)

data/README.md

@@ -13,7 +13,7 @@ Logstash provides infrastructure to automatically generate documentation for thi
 
 ## Need Help?
 
-Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
 
 ## Developing
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -94,23 +94,43 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   config :routing, :validate => :string
 
   # The name of your cluster if you set it on the Elasticsearch side. Useful
-  # for discovery.
+  # for discovery when using `node` or `transport` protocols.
+  # By default, it looks for a cluster named 'elasticsearch'.
   config :cluster, :validate => :string
 
-  # The hostname or IP address of the host to use for Elasticsearch unicast discovery
-  # This is only required if the normal multicast/cluster discovery stuff won't
-  # work in your environment.
-  #
-  # The plugin will use multicast discovery to connect to Elasticsearch
-  # when using `protocol => node` without setting a host. When setting unicast 
-  # hosts for `node` protocol, it is important to confirm that at least one non-client
-  # node is listed in the `:host` list. client nodes are filtered by default in zen-discovery.
-  # You need to set `discovery.zen.master_election.filter_client` to `false` to override this behavior.
-  #
-  # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-zen.html#multicast[Multicast Discovery Docs]
-  #
+  # For the `node` protocol, if you do not specify `host`, it will attempt to use
+  # multicast discovery to connect to Elasticsearch.  If http://www.elastic.co/guide/en/elasticsearch/guide/current/_important_configuration_changes.html#_prefer_unicast_over_multicast[multicast is disabled] in Elasticsearch, 
+  # you must include the hostname or IP address of the host(s) to use for Elasticsearch unicast discovery.
+  # Remember the `node` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html#modules-transport[transport] address (eg. 9300, not 9200).
+  #     `"127.0.0.1"`
+  #     `["127.0.0.1:9300","127.0.0.2:9300"]`
+  # When setting hosts for `node` protocol, it is important to confirm that at least one non-client
+  # node is listed in the `host` list.  Also keep in mind that the `host` parameter when used with 
+  # the `node` protocol is for *discovery purposes only* (not for load balancing).  When multiple hosts 
+  # are specified, it will contact the first host to see if it can use it to discover the cluster.  If not, 
+  # then it will contact the second host in the list and so forth. With the `node` protocol, 
+  # Logstash will join the Elasticsearch cluster as a node client (which has a copy of the cluster
+  # state) and this node client is the one that will automatically handle the load balancing of requests 
+  # across data nodes in the cluster.  
+  # If you are looking for a high availability setup, our recommendation is to use the `transport` protocol (below), 
+  # set up multiple http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[client nodes] and list the client nodes in the `host` parameter.
+  # 
+  # For the `transport` protocol, it will load balance requests across the hosts specified in the `host` parameter.
+  # Remember the `transport` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html#modules-transport[transport] address (eg. 9300, not 9200).
   #     `"127.0.0.1"`
   #     `["127.0.0.1:9300","127.0.0.2:9300"]`
+  # There is also a `sniffing` option (see below) that can be used with the transport protocol to instruct it to use the host to sniff for
+  # "alive" nodes in the cluster and automatically use it as the hosts list (but will skip the dedicated master nodes).  
+  # If you do not use the sniffing option, it is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `host` list
+  # to prevent Logstash from sending bulk requests to the master nodes. So this parameter should only reference either data or client nodes.
+  #
+  # For the `http` protocol, it will load balance requests across the hosts specified in the `host` parameter.
+  # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
+  #     `"127.0.0.1"`
+  #     `["127.0.0.1:9200","127.0.0.2:9200"]`
+  # It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `host` list
+  # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes.
+
   config :host, :validate => :array
 
   # The port for Elasticsearch transport to use.
@@ -171,12 +191,16 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
   # Choose the protocol used to talk to Elasticsearch.
   #
-  # The 'node' protocol will connect to the cluster as a normal Elasticsearch
-  # node (but will not store data). This allows you to use things like
-  # multicast discovery. If you use the `node` protocol, you must permit
+  # The 'node' protocol (default) will connect to the cluster as a normal Elasticsearch
+  # node (but will not store data). If you use the `node` protocol, you must permit
   # bidirectional communication on the port 9300 (or whichever port you have
   # configured).
   #
+  # If you do not specify the `host` parameter, it will use  multicast for http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery-zen.html[Elasticsearch discovery].  While this may work in a test/dev environment where multicast is enabled in 
+  # Elasticsearch, we strongly recommend http://www.elastic.co/guide/en/elasticsearch/guide/current/_important_configuration_changes.html#_prefer_unicast_over_multicast[disabling multicast]
+  # in Elasticsearch.  To connect to an Elasticsearch cluster with multicast disabled,
+  # you must include the `host` parameter (see relevant section above).  
+  #
   # The 'transport' protocol will connect to the host you specify and will
   # not show up as a 'node' in the Elasticsearch cluster. This is useful
   # in situations where you cannot permit connections outbound from the
@@ -216,6 +240,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Enable SSL
   config :ssl, :validate => :boolean, :default => false
 
+  # Validate the server's certificate
+  # Disabling this severely compromises security
+  # For more information read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
+  config :ssl_certificate_verification, :validate => :boolean, :default => true
+
   # The .cer or .pem file to validate the server's certificate
   config :cacert, :validate => :path
 
@@ -533,7 +562,15 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     elsif @truststore
       ssl_options[:truststore_password] = @truststore_password.value if @truststore_password
     end
-    ssl_options[:truststore] = @truststore
+    ssl_options[:truststore] = @truststore if @truststore
+    if @ssl_certificate_verification == false
+      @logger.warn [
+        "** WARNING ** Detected UNSAFE options in elasticsearch output configuration!",
+        "** WARNING ** You have enabled encryption but DISABLED certificate verification.",
+        "** WARNING ** To make sure your data is secure change :ssl_certificate_verification to true"
+      ].join("\n")
+      ssl_options[:verify] = false
+    end
     { ssl: ssl_options }
   end
 

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '0.2.7'
+  s.version         = '0.2.8'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "Output events to elasticsearch"

data/spec/outputs/elasticsearch_spec.rb

@@ -900,6 +900,34 @@ describe "ship lots of events w/ default index_type and dynamic routing key usin
         end
       end
     end
+
+    context "when using http protocol" do
+      protocol = "http"
+      context "when using ssl without cert verification" do
+        subject do
+          require "logstash/outputs/elasticsearch"
+          settings = {
+            "protocol" => protocol,
+            "host" => "node01",
+            "ssl" => true,
+            "ssl_certificate_verification" => false
+          }
+          next LogStash::Outputs::ElasticSearch.new(settings)
+        end
+
+        it "should pass the flag to the ES client" do
+          expect(::Elasticsearch::Client).to receive(:new) do |args|
+            expect(args[:ssl]).to eq(:verify => false)
+          end
+          subject.register
+        end
+
+        it "print a warning" do
+          expect(subject.logger).to receive(:warn)
+          subject.register
+        end
+      end
+    end
   end
 
   describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.2.8
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-28 00:00:00.000000000 Z
+date: 2015-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -161,6 +161,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE