logstash-input-elasticsearch 4.12.3 → 4.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a4ed685c9f90446a47fc84cb4e52e585f164c5451c9dca3f7359b756077bbd4
-  data.tar.gz: 050b54e5c660a4ab436f57ab16ea20d6a64a6b8a8c063eaaca69cc2c77235bd4
+  metadata.gz: d87078bdf63844901c7684624bc15bd6af23c71c728adb20b86ba76738957926
+  data.tar.gz: 34d8ce81035665b93623aec9008c511000b80a463c0cf482fb459249c763aeaf
 SHA512:
-  metadata.gz: 902070db657622bd65bcba8c17470456366134aae23cd38fb70ec89c97686b74da591e6f47d34957f3e5753f0e029ad78b845dd6355d6d559b6c135e0c94d28e
-  data.tar.gz: 22391299a87f5ce0a3085b4be7886a7f7251c9e33bd7051996871ea7cc34cbd7fbc53a3d54d841722b319949c4422e0ab7e6ff7d04899db6255543363fc32f66
+  metadata.gz: 4b2c5aa7a229bdbc89df35276c22a72f1cdbefc03a9069dfb74e64aad64d121aeb3ee67a9b632fd8ab445d1a256cce23f67feabb4ff379ed78a15507f28b1dce
+  data.tar.gz: 7c7a45cd3817d9e746a03eb166293e389eee16fc512bb2e0ecc546158d67f1f4e0f54bc191212e02ce1af206fe5f98374d70defa3f8ef218bed9e210ac8a8a65

data/CHANGELOG.md

@@ -1,5 +1,8 @@
+## 4.13.0
+  - Added support for `ca_trusted_fingerprint` when run on Logstash 8.3+ [#178](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/178)
+
 ## 4.12.3
-  - Fix: update Elasticsearch Ruby client to correctly customize 'user-agent' header[#171](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/171)
+  - Fix: update Elasticsearch Ruby client to correctly customize 'user-agent' header [#171](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/171)
 
 ## 4.12.2
   - Fix: hosts => "es_host:port" regression [#168](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/168)
@@ -24,7 +27,6 @@
     header isn't passed, this leads to the plugin not being able to leverage `user`/`password` credentials set by the user.
     [#153](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/153)
 
-
 ## 4.9.1
   - [DOC] Replaced hard-coded links with shared attributes [#143](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/143)
   - [DOC] Added missing quote to docinfo_fields example [#145](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/145)

data/docs/index.asciidoc

@@ -103,6 +103,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-connect_timeout_seconds>> | <<number,number>>|No
@@ -152,6 +153,15 @@ API key API].
 
 SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary.
 
+[id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
+===== `ca_trusted_fingerprint`
+
+* Value type is <<string,string>>, and must contain exactly 64 hexadecimal characters.
+* There is no default value for this setting.
+* Use of this option _requires_ Logstash 8.3+
+
+The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster.
+
 [id="plugins-{type}s-{plugin}-cloud_auth"]
 ===== `cloud_auth`
 

data/lib/logstash/inputs/elasticsearch.rb

@@ -7,6 +7,7 @@ require 'logstash/plugin_mixins/validator_support/field_reference_validation_ada
 require 'logstash/plugin_mixins/event_support/event_factory_adapter'
 require 'logstash/plugin_mixins/ecs_compatibility_support'
 require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
 require "base64"
 
 require "elasticsearch"
@@ -192,6 +193,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # If set, the _source of each hit will be added nested under the target instead of at the top-level
   config :target, :validate => :field_reference
 
+  # config :ca_trusted_fingerprint, :validate => :sha_256_hex
+  include LogStash::PluginMixins::CATrustedFingerprintSupport
+
   def initialize(params={})
     super(params)
 
@@ -381,7 +385,13 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   end
 
   def setup_ssl
-    @ssl && @ca_file ? { :ssl  => true, :ca_file => @ca_file } : {}
+    ssl_options = {}
+
+    ssl_options[:ssl] = true if @ssl
+    ssl_options[:ca_file] = @ca_file if @ssl && @ca_file
+    ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint
+
+    ssl_options
   end
 
   def setup_hosts

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.12.3'
+  s.version         = '4.13.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
 
   s.add_runtime_dependency 'elasticsearch', '>= 7.17.1'
+  s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
 
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'
@@ -37,4 +38,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'timecop'
   s.add_development_dependency 'cabin', ['~> 0.6']
   s.add_development_dependency 'webrick'
+
+  # 3.8.0 has breaking changes WRT to joining, which break our specs
+  s.add_development_dependency 'rufus-scheduler', '~> 3.0.9'
 end

data/spec/fixtures/test_certs/ca.der.sha256

@@ -0,0 +1 @@
+195a7e7b1bc29f3d7913a918a44721704d27fa56facea0cd72a8093c7107c283

data/spec/inputs/elasticsearch_spec.rb

@@ -19,7 +19,14 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
   let(:queue) { Queue.new }
 
   before(:each) do
-     Elasticsearch::Client.send(:define_method, :ping) { } # define no-action ping method
+    Elasticsearch::Client.send(:define_method, :ping) { } # define no-action ping method
+  end
+
+  let(:base_config) do
+    {
+        'hosts' => ["localhost"],
+        'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+    }
   end
 
   context "register" do
@@ -78,14 +85,10 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
 
     let(:config) do
-      %q[
-        input {
-          elasticsearch {
-            hosts => ["localhost"]
-            query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-          }
-        }
-      ]
+      {
+          'hosts' => ["localhost"],
+          'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+      }
     end
 
     let(:mock_response) do
@@ -128,10 +131,11 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       expect(client).to receive(:ping)
     end
 
+    before { plugin.register }
+
     it 'creates the events from the hits' do
-      event = input(config) do |pipeline, queue|
-        queue.pop
-      end
+      plugin.run queue
+      event = queue.pop
 
       expect(event).to be_a(LogStash::Event)
       expect(event.get("message")).to eql [ "ohayo" ]
@@ -139,21 +143,16 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
     context 'when a target is set' do
       let(:config) do
-        %q[
-          input {
-            elasticsearch {
-              hosts => ["localhost"]
-              query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-              target => "[@metadata][_source]"
-            }
-          }
-        ]
+        {
+            'hosts' => ["localhost"],
+            'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+            'target' => "[@metadata][_source]"
+        }
       end
 
       it 'creates the event using the target' do
-        event = input(config) do |pipeline, queue|
-          queue.pop
-        end
+        plugin.run queue
+        event = queue.pop
 
         expect(event).to be_a(LogStash::Event)
         expect(event.get("[@metadata][_source][message]")).to eql [ "ohayo" ]
@@ -450,24 +449,21 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
       end
 
-      context 'with docinfo enabled' do
-        let(:config_metadata) do
-          %q[
-              input {
-                elasticsearch {
-                  hosts => ["localhost"]
-                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                  docinfo => true
-                }
-              }
-          ]
+      before do
+        if do_register
+          plugin.register
+          plugin.run queue
         end
+      end
 
-        it "provides document info under metadata" do
-          event = input(config_metadata) do |pipeline, queue|
-            queue.pop
-          end
+      let(:do_register) { true }
 
+      let(:event) { queue.pop }
+
+      context 'with docinfo enabled' do
+        let(:config) { base_config.merge 'docinfo' => true }
+
+        it "provides document info under metadata" do
           if ecs_select.active_mode == :disabled
             expect(event.get("[@metadata][_index]")).to eq('logstash-2014.10.12')
             expect(event.get("[@metadata][_type]")).to eq('logs')
@@ -479,123 +475,72 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           end
         end
 
-        it 'merges values if the `docinfo_target` already exist in the `_source` document' do
-          config_metadata_with_hash = %Q[
-              input {
-                elasticsearch {
-                  hosts => ["localhost"]
-                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                  docinfo => true
-                  docinfo_target => 'metadata_with_hash'
-                }
-              }
-          ]
+        context 'with docinfo_target' do
+          let(:config) { base_config.merge 'docinfo' => true, 'docinfo_target' => docinfo_target }
+          let(:docinfo_target) { 'metadata_with_hash' }
+
+          it 'merges values if the `docinfo_target` already exist in the `_source` document' do
+            expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
+            expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
+            expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+            expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
+          end
+
+          context 'non-existent' do
+            let(:docinfo_target) { 'meta' }
+
+            it 'should move the document information to the specified field' do
+              expect(event.get("[meta][_index]")).to eq('logstash-2014.10.12')
+              expect(event.get("[meta][_type]")).to eq('logs')
+              expect(event.get("[meta][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+            end
 
-          event = input(config_metadata_with_hash) do |pipeline, queue|
-            queue.pop
           end
 
-          expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
-          expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
-          expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-          expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
         end
 
         context 'if the `docinfo_target` exist but is not of type hash' do
-          let (:config) { {
-              "hosts" => ["localhost"],
-              "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
-              "docinfo" => true,
-              "docinfo_target" => 'metadata_with_string'
-          } }
-          it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
+          let(:config) { base_config.merge 'docinfo' => true, "docinfo_target" => 'metadata_with_string' }
+          let(:do_register) { false }
+
+          it 'raises an exception if the `docinfo_target` exist but is not of type hash' do
             expect(client).not_to receive(:clear_scroll)
             plugin.register
             expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
           end
-        end
-
-        it 'should move the document information to the specified field' do
-          config = %q[
-              input {
-                elasticsearch {
-                  hosts => ["localhost"]
-                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                  docinfo => true
-                  docinfo_target => 'meta'
-                }
-              }
-          ]
-          event = input(config) do |pipeline, queue|
-            queue.pop
-          end
 
-          expect(event.get("[meta][_index]")).to eq('logstash-2014.10.12')
-          expect(event.get("[meta][_type]")).to eq('logs')
-          expect(event.get("[meta][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
         end
 
-        it "allows to specify which fields from the document info to save to metadata" do
-          fields = ["_index"]
-          config = %Q[
-              input {
-                elasticsearch {
-                  hosts => ["localhost"]
-                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                  docinfo => true
-                  docinfo_fields => #{fields}
-                }
-              }]
+        context 'with docinfo_fields' do
+          let(:config) { base_config.merge 'docinfo' => true, "docinfo_fields" => ["_index"] }
 
-          event = input(config) do |pipeline, queue|
-            queue.pop
+          it "allows to specify which fields from the document info to save to metadata" do
+            meta_base = event.get(ecs_select.active_mode == :disabled ? "@metadata" : "[@metadata][input][elasticsearch]")
+            expect(meta_base.keys).to eql ["_index"]
           end
 
-          meta_base = event.get(ecs_select.active_mode == :disabled ? "@metadata" : "[@metadata][input][elasticsearch]")
-          expect(meta_base.keys).to eq(fields)
         end
 
-        it 'should be able to reference metadata fields in `add_field` decorations' do
-          config = %q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-                add_field => {
-                  'identifier' => "foo:%{[@metadata][_type]}:%{[@metadata][_id]}"
-                }
-              }
-            }
-          ]
+        context 'add_field' do
+          let(:config) { base_config.merge 'docinfo' => true,
+                                           'add_field' => { 'identifier' => "foo:%{[@metadata][_type]}:%{[@metadata][_id]}" } }
 
-          event = input(config) do |pipeline, queue|
-            queue.pop
-          end
+          it 'should be able to reference metadata fields in `add_field` decorations' do
+            expect(event.get('identifier')).to eq('foo:logs:C5b2xLQwTZa76jBmHIbwHQ')
+          end if ecs_select.active_mode == :disabled
 
-          expect(event.get('identifier')).to eq('foo:logs:C5b2xLQwTZa76jBmHIbwHQ')
-        end if ecs_select.active_mode == :disabled
+        end
 
       end
 
-    end
+      context "when not defining the docinfo" do
+        let(:config) { base_config }
 
-    context "when not defining the docinfo" do
-      it 'should keep the document information in the root of the event' do
-        config = %q[
-          input {
-            elasticsearch {
-              hosts => ["localhost"]
-              query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-            }
-          }
-        ]
-        event = input(config) do |pipeline, queue|
-          queue.pop
+        it 'should keep the document information in the root of the event' do
+          expect(event.get("[@metadata]")).to be_empty
         end
-
-        expect(event.get("[@metadata]")).to be_empty
       end
+
     end
   end
 
@@ -740,9 +685,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
             begin
               @server = WEBrick::HTTPServer.new :Port => 0, :DocumentRoot => ".",
                        :Logger => Cabin::Channel.get, # silence WEBrick logging
-                       :StartCallback => Proc.new {
-                             queue.push("started")
-                           }
+                       :StartCallback => Proc.new { queue.push("started") }
               @port = @server.config[:Port]
               @server.mount_proc '/' do |req, res|
                 res.body = '''
@@ -811,11 +754,9 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
                 @first_req_waiter.countDown()
               end
 
-
-
               @server.start
             rescue => e
-              puts "Error in webserver thread #{e}"
+              warn "ERROR in webserver thread #{e.inspect}\n  #{e.backtrace.join("\n  ")}"
               # ignore
             end
           end
@@ -914,6 +855,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
           plugin.register
         end
+
+        after { plugin.do_stop }
       end
     end
 
@@ -942,13 +885,16 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
 
     it "should properly schedule" do
-      expect(plugin).to receive(:do_run) {
-        queue << LogStash::Event.new({})
-      }.at_least(:twice)
-      runner = Thread.start { plugin.run(queue) }
-      sleep 3.0
-      plugin.stop
-      runner.join
+      begin
+        expect(plugin).to receive(:do_run) {
+          queue << LogStash::Event.new({})
+        }.at_least(:twice)
+        runner = Thread.start { plugin.run(queue) }
+        sleep 3.0
+      ensure
+        plugin.do_stop
+        runner.join if runner
+      end
       expect(queue.size).to be >= 2
     end
 

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -72,25 +72,45 @@ describe LogStash::Inputs::Elasticsearch do
 
   describe 'against a secured elasticsearch', secure_integration: true do
 
+    # client_options is for an out-of-band helper
     let(:client_options) { { :ca_file => ca_file, :user => user, :password => password } }
 
-    let(:config) { super().merge('user' => user, 'password' => password, 'ssl' => true, 'ca_file' => ca_file) }
+    let(:config) { super().merge('user' => user, 'password' => password) }
 
-    it_behaves_like 'an elasticsearch index plugin'
+    shared_examples 'secured_elasticsearch' do
+      it_behaves_like 'an elasticsearch index plugin'
 
-    context "incorrect auth credentials" do
+      context "incorrect auth credentials" do
 
-      let(:config) do
-        super().merge('user' => 'archer', 'password' => 'b0gus!')
-      end
+        let(:config) do
+          super().merge('user' => 'archer', 'password' => 'b0gus!')
+        end
 
-      let(:queue) { [] }
+        let(:queue) { [] }
 
-      it "fails to run the plugin" do
-        expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
+        it "fails to run the plugin" do
+          expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
+        end
       end
     end
 
+    context 'with ca_file' do
+      let(:config) { super().merge('ssl' => true, 'ca_file' => ca_file) }
+      it_behaves_like 'secured_elasticsearch'
+    end
+
+    context 'with `ca_trusted_fingerprint`' do
+      let(:ca_trusted_fingerprint) { File.read("spec/fixtures/test_certs/ca.der.sha256").chomp }
+      let(:config) { super().merge('ssl' => true, 'ca_trusted_fingerprint' => ca_trusted_fingerprint) }
+
+      if Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create("8.3.0")
+        it_behaves_like 'secured_elasticsearch'
+      else
+        it 'raises a configuration error' do
+          expect { plugin }.to raise_exception(LogStash::ConfigurationError, a_string_including("ca_trusted_fingerprint"))
+        end
+      end
+    end
   end
 
   context 'setting host:port', integration: true do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.12.3
+  version: 4.13.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-29 00:00:00.000000000 Z
+date: 2022-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -86,6 +86,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.17.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-ca_trusted_fingerprint_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -212,6 +226,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.9
+  name: rufus-scheduler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.9
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program
@@ -233,6 +261,7 @@ files:
 - logstash-input-elasticsearch.gemspec
 - spec/es_helper.rb
 - spec/fixtures/test_certs/ca.crt
+- spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
@@ -266,6 +295,7 @@ summary: Reads query results from an Elasticsearch cluster
 test_files:
 - spec/es_helper.rb
 - spec/fixtures/test_certs/ca.crt
+- spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key