logstash-output-elasticsearch 6.2.3-java → 6.2.4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 23e44847bd8662bb97c4ceb1a3e502ad6883adbd
-  data.tar.gz: 012dbb55afac5bdcb401df61a48aedb531a84d15
+  metadata.gz: c2ae1de8b2ef04b483cc7ea7d90a9abe2d23493b
+  data.tar.gz: ccbbb21f0a3c56a03f7a4c209d1cdbe80aa8706d
 SHA512:
-  metadata.gz: eab7f09f2abca0a75f47424eb3f01f80c8a6f968717bfd04a75bb0dcc14cba0a7f75a73cfb8ae7aa78318be6ca6d22d1b0d4b5611b282f8262a26a8cb026b66b
-  data.tar.gz: 7707d7d2929775fa188dcefd32602307f01e6921bb5b7be5cd07ee56db2a787712640d21bc007bc14ecc87277e3bfc4686b712103cb935527fa2042966a6eb73
+  metadata.gz: 0b894a22dc7344fa44b9dd8f3a14f3575be12e31c982ef1424f915094bfb69d7378492b17b8bc6357a0b8fce2379a8691b30a6294ce5de7fc2b04ff404f7f217
+  data.tar.gz: a888417183f7851d31b0cfc567bb0c0c0d8285c518224c715cfd82db96d3e4b2c2bf5c1026f49a82d91ba0223a8f4aa4d61acb0f83cff809673db8e461525dea

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 6.2.4
+- Fix bug where using escaped characters in the password field would attempt to show a warning but instead crash.
+  The warning was also not necessary since escaped characters never worked there before.
+
 ## 6.2.3
 - Fixed a bug introduced in 6.2.2 where passwords needing escapes were not actually sent to ES properly
   encoded. 

data/README.md

@@ -47,14 +47,10 @@ bundle exec rspec
 
 - Run integration tests
 
-Dependencies: [Docker](http://docker.com)
-
-Before the test suite is run, we will load and run an
-Elasticsearch instance within a docker container. This container 
-will be cleaned up when suite has finished.
-
 ```sh
-bundle exec rspec --tag integration
+export INTEGRATION=true
+export ES_VERSION=5.1.1
+./travis-run.sh 
 ```
 
 ### 2. Running your unpublished Plugin in Logstash

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -124,20 +124,6 @@ module LogStash; module Outputs; class ElasticSearch;
       unsafe_password = password && password.value
       unsafe_escaped_password = unsafe_password ? CGI.escape(unsafe_password) : nil
       
-      # TODO: Remove this when we release LS6.0.0
-      if unsafe_password =~ /%[0-9A-Fa-f]{2}/
-        m <<-EOM
-        The Elasticsearch output was provided a password that looks like it includes URL encoded characters.
-        Previous versions of this plugin had a bug that required a workaround where users needed to manually
-        URL encode special characters in the password field. Given this, URL encoded strings will 
-        be doubly escaped making authentication fail. This may not apply to you.
-        If your password just happens to include string parts that simply look 
-        like URL encoded strings like '%2F' but are in fact just a part of your 
-        password then you can safely ignore this message. 
-        EOM
-        @logger.warn(m)
-      end
-      
       return {} unless user && unsafe_escaped_password
 
       {

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '6.2.3'
+  s.version         = '6.2.4'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/fixtures/htpasswd

@@ -0,0 +1,2 @@
+fancyuser:$apr1$Eq3/Qh40$MRzg6mccKUVmx8HJvlqkK1
+simpleuser:$apr1$hQQ4QWmo$ECyA1DFO3iCRs07zVXqAq1

data/spec/fixtures/nginx_reverse_proxy.conf

@@ -0,0 +1,22 @@
+worker_processes  1;
+daemon off; # run in foreground
+
+events {
+  worker_connections 1024;
+}
+
+http {
+  server {
+    listen 9900 default_server;
+    ssl on;
+    ssl_certificate server.crt;
+    ssl_certificate_key server.key;
+    client_max_body_size 200m;
+
+    location / {
+      proxy_pass http://localhost:9200;
+      auth_basic "Restricted Content";
+      auth_basic_user_file htpasswd;
+    }
+  }
+}

data/spec/integration/outputs/index_spec.rb

@@ -1,47 +1,6 @@
 require_relative "../../../spec/es_spec_helper"
 require "logstash/outputs/elasticsearch"
 
-shared_examples "an indexer" do
-    let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
-    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
-    let(:event_count) { 10000 + rand(500) }
-    let(:config) { "not implemented" }
-    let(:events) { event_count.times.map { event }.to_a }
-    subject { LogStash::Outputs::ElasticSearch.new(config) }
-
-    before do
-      subject.register
-    end
-
-    it "ships events" do
-      subject.multi_receive(events)
-      index_url = "http://#{get_host_port}/#{index}"
-
-      ftw = FTW::Agent.new
-      ftw.post!("#{index_url}/_refresh")
-
-      # Wait until all events are available.
-      Stud::try(10.times) do
-        data = ""
-        response = ftw.get!("#{index_url}/_count?q=*")
-        response.read_body { |chunk| data << chunk }
-        result = LogStash::Json.load(data)
-        cur_count = result["count"]
-        insist { cur_count } == event_count
-      end
-
-      response = ftw.get!("#{index_url}/_search?q=*&size=1000")
-      data = ""
-      response.read_body { |chunk| data << chunk }
-      result = LogStash::Json.load(data)
-      result["hits"]["hits"].each do |doc|
-        insist { doc["_type"] } == type
-        insist { doc["_index"] } == index
-      end
-    end
-end
-
 describe "TARGET_BULK_BYTES", :integration => true do
   let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
   let(:event_count) { 1000 }
@@ -56,7 +15,6 @@ describe "TARGET_BULK_BYTES", :integration => true do
   let(:type) { 10.times.collect { rand(10).to_s }.join("") }
   subject { LogStash::Outputs::ElasticSearch.new(config) }
 
-
   before do
     subject.register
     allow(subject.client).to receive(:bulk_send).with(any_args).and_call_original
@@ -86,20 +44,57 @@ describe "TARGET_BULK_BYTES", :integration => true do
   end
 end
 
+describe "indexing" do
+  let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:event_count) { 10000 + rand(500) }
+  let(:config) { "not implemented" }
+  let(:events) { event_count.times.map { event }.to_a }
+  subject { LogStash::Outputs::ElasticSearch.new(config) }
+  
+  let(:es_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{es_url}/#{index}"}
+  let(:http_client_options) { {} }
+  let(:http_client) do
+    Manticore::Client.new(http_client_options)
+  end
 
-describe "an indexer with custom index_type", :integration => true do
-  it_behaves_like "an indexer" do
+  before do
+    subject.register
+  end
+  
+  shared_examples "an indexer" do
+    it "ships events" do
+      subject.multi_receive(events)
+
+      http_client.post("#{es_url}/_refresh").call
+
+      response = http_client.get("#{index_url}/_count?q=*")
+      result = LogStash::Json.load(response.body)
+      cur_count = result["count"]
+      expect(cur_count).to eq(event_count)
+
+      response = http_client.get("#{index_url}/_search?q=*&size=1000")
+      result = LogStash::Json.load(response.body)
+      result["hits"]["hits"].each do |doc|
+        expect(doc["_type"]).to eq(type)
+        expect(doc["_index"]).to eq(index)
+      end
+    end
+  end
+
+  describe "an indexer with custom index_type", :integration => true do
     let(:config) {
       {
         "hosts" => get_host_port,
         "index" => index
       }
     }
+    it_behaves_like("an indexer")
   end
-end
 
-describe "an indexer with no type value set (default to logs)", :integration => true do
-  it_behaves_like "an indexer" do
+  describe "an indexer with no type value set (default to logs)", :integration => true do
     let(:type) { "logs" }
     let(:config) {
       {
@@ -107,5 +102,60 @@ describe "an indexer with no type value set (default to logs)", :integration =>
         "index" => index
       }
     }
+    it_behaves_like("an indexer")
+  end
+
+  describe "a secured indexer", :integration => true do
+    let(:user) { "simpleuser" }
+    let(:password) { "abc123" }
+    let(:cacert) { "spec/fixtures/server.crt" }
+    let(:es_url) {"https://localhost:9900"}
+    let(:config) do
+      {
+        "hosts" => ["localhost:9900"],
+        "user" => user,
+        "password" => password,
+        "ssl" => true,
+        "cacert" => "spec/fixtures/server.crt",
+        "index" => index
+      }
+    end
+    let(:http_client_options) do
+      {
+        :auth => {
+          :user => user,
+          :password => password
+        }, 
+        :ssl => {
+          :enabled => true,
+          :ca_file => cacert
+        }
+      }
+    end
+    it_behaves_like("an indexer") 
+    
+    describe "with a password requiring escaping" do
+      let(:user) { "fancyuser" }
+      let(:password) { "ab%12#" }
+      
+      include_examples("an indexer")
+    end
+    
+    describe "with a password requiring escaping in the URL" do
+      let(:config) do
+        {
+          "hosts" => ["https://#{user}:#{CGI.escape(password)}@localhost:9900"],
+          "ssl" => true,
+          "cacert" => "spec/fixtures/server.crt",
+          "index" => index
+        }
+      end
+      
+      begin
+        include_examples("an indexer")
+      rescue => e
+        require 'pry'; binding.pry
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 6.2.3
+  version: 6.2.4
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-20 00:00:00.000000000 Z
+date: 2017-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -206,6 +206,8 @@ files:
 - lib/logstash/outputs/elasticsearch/template_manager.rb
 - logstash-output-elasticsearch.gemspec
 - spec/es_spec_helper.rb
+- spec/fixtures/htpasswd
+- spec/fixtures/nginx_reverse_proxy.conf
 - spec/fixtures/scripts/scripted_update.groovy
 - spec/fixtures/scripts/scripted_update_nested.groovy
 - spec/fixtures/scripts/scripted_upsert.groovy
@@ -217,7 +219,6 @@ files:
 - spec/integration/outputs/pipeline_spec.rb
 - spec/integration/outputs/retry_spec.rb
 - spec/integration/outputs/routing_spec.rb
-- spec/integration/outputs/secure_spec.rb
 - spec/integration/outputs/sniffer_spec.rb
 - spec/integration/outputs/templates_5x_spec.rb
 - spec/integration/outputs/templates_spec.rb
@@ -259,6 +260,8 @@ specification_version: 4
 summary: Logstash Output to Elasticsearch
 test_files:
 - spec/es_spec_helper.rb
+- spec/fixtures/htpasswd
+- spec/fixtures/nginx_reverse_proxy.conf
 - spec/fixtures/scripts/scripted_update.groovy
 - spec/fixtures/scripts/scripted_update_nested.groovy
 - spec/fixtures/scripts/scripted_upsert.groovy
@@ -270,7 +273,6 @@ test_files:
 - spec/integration/outputs/pipeline_spec.rb
 - spec/integration/outputs/retry_spec.rb
 - spec/integration/outputs/routing_spec.rb
-- spec/integration/outputs/secure_spec.rb
 - spec/integration/outputs/sniffer_spec.rb
 - spec/integration/outputs/templates_5x_spec.rb
 - spec/integration/outputs/templates_spec.rb

data/spec/integration/outputs/secure_spec.rb

@@ -1,104 +0,0 @@
-require_relative "../../../spec/es_spec_helper"
-
-describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do
-  subject do
-    require "logstash/outputs/elasticsearch"
-    settings = {
-      "node_name" => "logstash",
-      "cluster" => "elasticsearch",
-      "hosts" => "node01",
-      "user" => "user",
-      "password" => "changeme",
-      "ssl" => true,
-      "cacert" => "/tmp/ca/certs/cacert.pem",
-      # or
-      #"truststore" => "/tmp/ca/truststore.jks",
-      #"truststore_password" => "testeteste"
-    }
-    next LogStash::Outputs::ElasticSearch.new(settings)
-  end
-
-  before :each do
-    subject.register
-  end
-
-  it "sends events to ES" do
-    expect {
-      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-    }.to_not raise_error
-  end
-end
-
-describe "connect using HTTP Authentication", :elasticsearch_secure => true do
-  subject do
-    require "logstash/outputs/elasticsearch"
-    settings = {
-      "cluster" => "elasticsearch",
-      "hosts" => "node01",
-      "user" => "user",
-      "password" => "changeme",
-    }
-    next LogStash::Outputs::ElasticSearch.new(settings)
-  end
-
-  before :each do
-    subject.register
-  end
-
-  it "sends events to ES" do
-    expect {
-      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-    }.to_not raise_error
-  end
-end
-
-describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do
-  subject do
-    require "logstash/outputs/elasticsearch"
-    settings = {
-      "node_name" => "logstash",
-      "cluster" => "elasticsearch",
-      "hosts" => "node01",
-      "user" => "user",
-      "password" => "changeme",
-      "ssl" => true,
-      "cacert" => "/tmp/ca/certs/cacert.pem",
-      # or
-      #"truststore" => "/tmp/ca/truststore.jks",
-      #"truststore_password" => "testeteste"
-    }
-    next LogStash::Outputs::ElasticSearch.new(settings)
-  end
-
-  before :each do
-    subject.register
-  end
-
-  it "sends events to ES" do
-    expect {
-      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-    }.to_not raise_error
-  end
-end
-
-describe "connect using HTTP Authentication", :elasticsearch_secure => true do
-  subject do
-    require "logstash/outputs/elasticsearch"
-    settings = {
-      "hosts" => "node01",
-      "user" => "user",
-      "password" => "changeme",
-    }
-    next LogStash::Outputs::ElasticSearch.new(settings)
-  end
-
-  before :each do
-    subject.register
-  end
-
-  it "sends events to ES" do
-    expect {
-      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-    }.to_not raise_error
-  end
-end