logstash-input-elasticsearch 4.10.0 → 4.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 84fb0092d51b303586c275c6cfaef3222391d7aabe910450a989291b691930c0
-  data.tar.gz: 955e27896dff25ec6568685f91cc7e34252bc54fbb5182133db1e4dd273efffe
+  metadata.gz: 5e0b94cdf348e1d25d3007f0d98637e11fea82606ac8116962f999d8743cd0f3
+  data.tar.gz: 82f489d3d852073644e488e0ced799d676f33e2c496d2353cbad911e87311b21
 SHA512:
-  metadata.gz: 2be653a935384617b32905f441060326f2b45eadcaacbf727d1ebe5c82711e3e9f0f3a038b9ca39aad6377e67bb5caf78c8b83a625d82aa8fca267b738dc9cab
-  data.tar.gz: a19916b987c434dfeb396379fa9f552fd001901bc5b37ce60407e4b7df69f5bd3ff76d4122b2f71d27c5d37d983a1dd283dcdf3aa1693730e93c2875f2459a1a
+  metadata.gz: 9be021ff84622606a6e3959055e4180cad9976dece9a690d732c289186243de051c5493d2c513e665c4dbdc5cdfbd68a42dcea80b4bb2f182d44b9b807fd5a7d
+  data.tar.gz: 5344d73f5b88542f03d045f45a630eb36d031f09fa2a79cfd50db4b2703350f91191c500e4c86adc26cb04d5c4ec4594c13eb2e7581edbe41d8e94866e077026

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.11.0
+  - Feat: add user-agent header passed to the Elasticsearch HTTP connection [#158](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/158)
+
 ## 4.10.0
   - Feat: added ecs_compatibility + event_factory support [#149](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/149)
 

data/lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb

@@ -20,7 +20,17 @@ if es_client_version >= Gem::Version.new('7.2') && es_client_version < Gem::Vers
             def apply_headers(request_options, options)
               headers = (options && options[:headers]) || {}
               headers[CONTENT_TYPE_STR] = find_value(headers, CONTENT_TYPE_REGEX) || DEFAULT_CONTENT_TYPE
-              headers[USER_AGENT_STR] = find_value(headers, USER_AGENT_REGEX) || user_agent_header
+
+              # this code is necessary to grab the correct user-agent header
+              # when this method is invoked with apply_headers(@request_options, options)
+              # from https://github.com/elastic/elasticsearch-ruby/blob/v7.14.0/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb#L113-L114
+              transport_user_agent = nil
+              if (options && options[:transport_options] && options[:transport_options][:headers])
+                transport_headers = options[:transport_options][:headers]
+                transport_user_agent = find_value(transport_headers, USER_AGENT_REGEX)
+              end
+
+              headers[USER_AGENT_STR] = transport_user_agent || find_value(headers, USER_AGENT_REGEX) || user_agent_header
               headers[ACCEPT_ENCODING] = GZIP if use_compression?
               (request_options[:headers] ||= {}).merge!(headers) # this line was changed
             end

data/lib/logstash/inputs/elasticsearch.rb

@@ -222,6 +222,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     transport_options = {:headers => {}}
     transport_options[:headers].merge!(setup_basic_auth(user, password))
     transport_options[:headers].merge!(setup_api_key(api_key))
+    transport_options[:headers].merge!({'user-agent' => prepare_user_agent()})
     transport_options[:request_timeout] = @request_timeout_seconds unless @request_timeout_seconds.nil?
     transport_options[:connect_timeout] = @connect_timeout_seconds unless @connect_timeout_seconds.nil?
     transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?
@@ -407,6 +408,18 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     { 'Authorization' => "ApiKey #{token}" }
   end
 
+  def prepare_user_agent
+      os_name = java.lang.System.getProperty('os.name')
+      os_version = java.lang.System.getProperty('os.version')
+      os_arch = java.lang.System.getProperty('os.arch')
+      jvm_vendor = java.lang.System.getProperty('java.vendor')
+      jvm_version = java.lang.System.getProperty('java.version')
+
+      plugin_version = Gem.loaded_specs["logstash-input-elasticsearch"].version
+      # example: logstash/7.14.1 (OS=Linux-5.4.0-84-generic-amd64; JVM=AdoptOpenJDK-11.0.11) logstash-input-elasticsearch/4.10.0
+      "logstash/#{LOGSTASH_VERSION} (OS=#{os_name}-#{os_version}-#{os_arch}; JVM=#{jvm_vendor}-#{jvm_version}) logstash-#{@plugin_type}-#{config_name}/#{plugin_version}"
+  end
+
   def fill_user_password_from_cloud_auth
     return unless @cloud_auth
 

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.10.0'
+  s.version         = '4.11.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -36,4 +36,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'faraday', "~> 0.15.4"
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'timecop'
+  s.add_development_dependency 'cabin', ['~> 0.6']
+  s.add_development_dependency 'webrick'
 end

data/spec/inputs/elasticsearch_spec.rb

@@ -7,6 +7,9 @@ require "timecop"
 require "stud/temporary"
 require "time"
 require "date"
+require "cabin"
+require "webrick"
+require "uri"
 
 require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 
@@ -690,6 +693,156 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       end
     end
 
+    class StoppableServer
+
+      attr_reader :port
+
+      def initialize()
+        queue = Queue.new
+        @first_req_waiter = java.util.concurrent.CountDownLatch.new(1)
+        @first_request = nil
+
+        @t = java.lang.Thread.new(
+          proc do
+            begin
+              @server = WEBrick::HTTPServer.new :Port => 0, :DocumentRoot => ".",
+                       :Logger => Cabin::Channel.get, # silence WEBrick logging
+                       :StartCallback => Proc.new {
+                             queue.push("started")
+                           }
+              @port = @server.config[:Port]
+              @server.mount_proc '/' do |req, res|
+                res.body = '''
+                {
+                    "name": "ce7ccfb438e8",
+                    "cluster_name": "docker-cluster",
+                    "cluster_uuid": "DyR1hN03QvuCWXRy3jtb0g",
+                    "version": {
+                        "number": "7.13.1",
+                        "build_flavor": "default",
+                        "build_type": "docker",
+                        "build_hash": "9a7758028e4ea59bcab41c12004603c5a7dd84a9",
+                        "build_date": "2021-05-28T17:40:59.346932922Z",
+                        "build_snapshot": false,
+                        "lucene_version": "8.8.2",
+                        "minimum_wire_compatibility_version": "6.8.0",
+                        "minimum_index_compatibility_version": "6.0.0-beta1"
+                    },
+                    "tagline": "You Know, for Search"
+                }
+                '''
+                res.status = 200
+                res['Content-Type'] = 'application/json'
+                @first_request = req
+                @first_req_waiter.countDown()
+              end
+
+              @server.mount_proc '/logstash_unit_test/_search' do |req, res|
+                res.body = '''
+                {
+                  "took" : 1,
+                  "timed_out" : false,
+                  "_shards" : {
+                    "total" : 1,
+                    "successful" : 1,
+                    "skipped" : 0,
+                    "failed" : 0
+                  },
+                  "hits" : {
+                    "total" : {
+                      "value" : 10000,
+                      "relation" : "gte"
+                    },
+                    "max_score" : 1.0,
+                    "hits" : [
+                      {
+                        "_index" : "test_bulk_index_2",
+                        "_type" : "_doc",
+                        "_id" : "sHe6A3wBesqF7ydicQvG",
+                        "_score" : 1.0,
+                        "_source" : {
+                          "@timestamp" : "2021-09-20T15:02:02.557Z",
+                          "message" : "{\"name\": \"Andrea\"}",
+                          "@version" : "1",
+                          "host" : "kalispera",
+                          "sequence" : 5
+                        }
+                      }
+                    ]
+                  }
+                }
+                '''
+                res.status = 200
+                res['Content-Type'] = 'application/json'
+                @first_request = req
+                @first_req_waiter.countDown()
+              end
+
+
+
+              @server.start
+            rescue => e
+              puts "Error in webserver thread #{e}"
+              # ignore
+            end
+          end
+        )
+        @t.daemon = true
+        @t.start
+        queue.pop # blocks until the server is up
+      end
+
+      def stop
+        @server.shutdown
+      end
+
+      def wait_receive_request
+        @first_req_waiter.await(2, java.util.concurrent.TimeUnit::SECONDS)
+        @first_request
+      end
+    end
+
+    describe "'user-agent' header" do
+      let!(:webserver) { StoppableServer.new } # webserver must be started before the call, so no lazy "let"
+
+      after :each do
+        webserver.stop
+      end
+
+      it "server should be started" do
+        require 'net/http'
+        response = nil
+        Net::HTTP.start('localhost', webserver.port) {|http|
+          response = http.request_get('/')
+        }
+        expect(response.code.to_i).to eq(200)
+      end
+
+      context "used by plugin" do
+        let(:config) do
+          {
+            "hosts" => ["localhost:#{webserver.port}"],
+            "query" => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }',
+            "index" => "logstash_unit_test"
+          }
+        end
+        let(:plugin) { described_class.new(config) }
+        let(:event)  { LogStash::Event.new({}) }
+
+        it "client should sent the expect user-agent" do
+          plugin.register
+
+          queue = []
+          plugin.run(queue)
+
+          request = webserver.wait_receive_request
+
+          expect(request.header['user-agent'].size).to eq(1)
+          expect(request.header['user-agent'][0]).to match(/logstash\/\d*\.\d*\.\d* \(OS=.*; JVM=.*\) logstash-input-elasticsearch\/\d*\.\d*\.\d*/)
+        end
+      end
+    end
+
     shared_examples'configurable timeout' do |config_name, manticore_transport_option|
       let(:config_value) { fail NotImplementedError }
       let(:config) { super().merge(config_name => config_value) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.10.0
+  version: 4.11.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-23 00:00:00.000000000 Z
+date: 2021-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -20,8 +20,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -37,8 +37,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.3'
   name: logstash-mixin-ecs_compatibility_support
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -51,8 +51,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-event_support
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -65,8 +65,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-validator_support
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -79,8 +79,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 5.0.5
   name: elasticsearch
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -93,8 +93,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: tzinfo
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -107,8 +107,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: tzinfo-data
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -121,8 +121,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: rufus-scheduler
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -135,8 +135,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.7.1
   name: manticore
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -149,8 +149,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-plain
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -163,8 +163,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.15.4
   name: faraday
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -177,8 +177,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -191,8 +191,36 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: timecop
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  name: cabin
+  prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: webrick
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -220,8 +248,6 @@ files:
 - spec/es_helper.rb
 - spec/fixtures/test_certs/ca.crt
 - spec/fixtures/test_certs/ca.key
-- spec/fixtures/test_certs/client/ls.crt
-- spec/fixtures/test_certs/client/ls.key
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
 - spec/inputs/elasticsearch_spec.rb
@@ -247,7 +273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Reads query results from an Elasticsearch cluster
@@ -255,8 +281,6 @@ test_files:
 - spec/es_helper.rb
 - spec/fixtures/test_certs/ca.crt
 - spec/fixtures/test_certs/ca.key
-- spec/fixtures/test_certs/client/ls.crt
-- spec/fixtures/test_certs/client/ls.key
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
 - spec/inputs/elasticsearch_spec.rb

data/spec/fixtures/test_certs/client/ls.crt

@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDMTCCAhmgAwIBAgIUCJ5+zdYJIlL04EOwC0tqVbZYKQUwDQYJKoZIhvcNAQEL
-BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
-cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNTI3WhcNMjQwODExMDUxNTI3WjANMQswCQYD
-VQQDEwJsczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMFY71j9Proa
-x95bWoBQOcc0ncqVvQO+Tk/Do4ynoe64pOJEf3txh1viHbnzFUGf+NF5WW6trlZ2
-ErP81vy7Ds5J5ngXjjdzOOGsTs9+l7KkqPfUwXqGoldWFtr//9mkLvWpd8uPvVtO
-dRnpcjQSlHHEB/HaqxkyBAvHLv1Fi1jTgIgn32NEM2mlCJ3M8OVfO9pqlO/6gjjs
-Miow8zZqtczeuv0JPu7V5xPDrcX0xh0kZdpH4gSh9r314TwZXFCofNUbkZrPV+Q9
-XJs64NlBjJZkd5qwKeXujZRV4eVvAOTlp+4Nh4eDqXE323s5yiPuov0StvNrbh3d
-rcg+IM+RTdMCAwEAAaNiMGAwHQYDVR0OBBYEFGnXHJEJ+LGmQTqDNr50C8FE8pcF
-MB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMBMGA1UdEQQMMAqCCGxv
-Z3N0YXNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEBAI08C6IeweN3LrEq
-ZauDVoiE2IdA1/nN3sxl+wL2xfauv1nctxej9TYR3mNoWiacgbbfJkPMCSIYk2Vc
-G396yLiGC1V96FfonnKfr3tKd0BiijTu3u5pOTgNNf5n4TZaTHTYmuKPtWoXyuLR
-QbH3jdgq9aq/9bwK0E5FOmuv6LGatnKzLf56aHjzerSZCnRw7V/1J/Yj3cy6TB1l
-9Lc7IAk4dGyrgwfKCuZOSzAtCWpOA/FfqCqMSuW6lrZ1zAXnk6VI3RkmBCuLE6kj
-aAjwORJHyiwBsMQbaYcQaXXjhguS+iHrnWdR0DFs9gHJSTuf6EoOeygdtMDutb2O
-4xfHWG8=
------END CERTIFICATE-----

data/spec/fixtures/test_certs/client/ls.key

@@ -1,27 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEAwVjvWP0+uhrH3ltagFA5xzSdypW9A75OT8OjjKeh7rik4kR/
-e3GHW+IdufMVQZ/40XlZbq2uVnYSs/zW/LsOzknmeBeON3M44axOz36XsqSo99TB
-eoaiV1YW2v//2aQu9al3y4+9W051GelyNBKUccQH8dqrGTIEC8cu/UWLWNOAiCff
-Y0QzaaUInczw5V872mqU7/qCOOwyKjDzNmq1zN66/Qk+7tXnE8OtxfTGHSRl2kfi
-BKH2vfXhPBlcUKh81RuRms9X5D1cmzrg2UGMlmR3mrAp5e6NlFXh5W8A5OWn7g2H
-h4OpcTfbeznKI+6i/RK282tuHd2tyD4gz5FN0wIDAQABAoIBAGLHVvC14PgfeoEl
-VuU7F2moffzj5z8kWMnzf3j6o4ZcmxBmQmMEq0zMBrfbcr6mRe5u+rvKy8isZf3C
-bOuNfZDyvGYaUrQNj7/r0g+78zB3Y0PKVFaOth28g8y7ATFl6f/j5qn+85TUTotA
-cvIbk+9TYWO0fblPjjWeO2l1wC1ObV0pFrXVzthFTQk0oxl3Gyq54P5rPp7QPOdM
-jB0lSAhvkEEA7nIZ0dF1zm3RuByahSQC45kYEp5TCC8SDwO3WCJ32Q/dhPw0y9Go
-gbIEY9QBFc8Rn3HDYGTSRPVVDM3HctDqzCihWOgFQlv718yCAZYq43fdOcy+6BY/
-UT3xRhECgYEA5wWcYD8bY5x0fk2ubxKJTE4Fm2HvUrexvLX00yF3sX9XqrsiBBC4
-14mB2yFEKQL+M/ZqLNV9IfJhfRmBffoRvf9lAuHhUrw4wGijERoO9xkmS3lGvzw+
-hqLIMnd986wyWC8FdU+dStDg8PC97xK4bg0tKYo9jmKf3kb0Ov2UWQ0CgYEA1kCK
-w30P3ZgYakXUjbwv3KoRXIVm6cIqgvm8iPh7B0GeTUCIgGBSNzJ8LVts/m6V7xtU
-RJ8RKw2gqqUTiFqrRKefr0APFv0YeVgZ7zK2EGLrL2rFWNZZQqA97Mh/5CwFh+zs
-1/IYuqRUVDaLq0sAPcZA5KEoO91eChBVK/RAyl8CgYAqgVPGOZY2e6DLZEuF0ClG
-yswpTJmV5IplKC1Fc1DsbXuZxBh8Gv+HWJt1z+cUjKJsuRfL6/O7/TaGp9y1av88
-r/LL1vd4G31tmVL3YI4EVLJBDK1Bnjn615RyBJ496R7SLsSYUu+jxk68xe6MQCuC
-xBXdILw2qFq1sORavjE/OQKBgQCJlwNGDX9t2Cn9vYCF0Q+Pjyv9FbKEdevlFsor
-0B76BvrJM6M1hiXmSqaSXj89mfjxh8RzGQ/mbSb7z20eyNNqEJes7N+D7N+Vta1Z
-/mALX+sXFWNM7MJ/1fZOpGf1OQwIQW/MMi4NVlDNkAXb6BtskG/GI3R6FWw53ElG
-I+Kj0wKBgQDUCfiYYSA9wYug53uvE0LirZwMQjWTbqxMO/S3H0Be8rf+BLo+YY5X
-fF6L/NaKxNZzs2Uu6+xYLXhSMimiv8412t7qnjK5DLwTZfnj7EuNxd44cZ4cXDPx
-JtD9GgPJ5V3js1gA2usPnKLYPw5Pp3ayZn1q4Tg9A5cextKnyOV/Fg==
------END RSA PRIVATE KEY-----