logstash-filter-elasticsearch 3.9.5 → 3.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3c42fd982d06ded966f625f3665120e64edc9b8585a2e5086214a14dd41f6e2a
-  data.tar.gz: 8411efb4ab1747adf5d824c39d098ccc64660c657916920b1bb4a194e8baf8ef
+  metadata.gz: 8e04ff9ca7407b8404764623b42f7ce8e754fe9ceafc6bde7cc064bc13c30723
+  data.tar.gz: 3b317df7d5bc1b11c912e8f9d6ec14c2ef260fa975412f2ebbf2baf84da547ba
 SHA512:
-  metadata.gz: c7be68bcfcd60db5d7b7c76ae484d382f8efb470abe5c33393330b1a2344b8b01e79f6a02038b2f37032783bd6abc045f44a12c504cac8b2ca1d534d110872a9
-  data.tar.gz: 57d1ab30bb38a717d3f1e8da8dc9b70fb2b1873bb8e2e482b67a0ae2396a32948842233d586cf8ea29831280ad4af7f284f73e5eef5bf0027a9f2040588cca4b
+  metadata.gz: 6dd245e044d1c06882885e776ba64463c1dbfc93d3c80ad03ad4009b6a3b6246e5a166dc11c09cafb58f2b05b3a6e485741aef6a2137edc227081b1e06bf9e3c
+  data.tar.gz: b9b211e1bf50bcb18d028e5322e78eaa3e113a034550eaae4ee3550b941e74235b5ed672417759eb35709e46693f7f602b8106ce13beb17bc6937eacbd991514

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.10.0
+  - Feat: add user-agent header passed to the Elasticsearch HTTP connection [#152](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/152)
+
 ## 3.9.5
   - Fixed SSL handshake hang indefinitely with proxy setup [#151](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/151)
 

data/lib/logstash/filters/elasticsearch/client.rb

@@ -16,10 +16,12 @@ module LogStash
         password = options.fetch(:password, nil)
         api_key = options.fetch(:api_key, nil)
         proxy = options.fetch(:proxy, nil)
+        user_agent = options[:user_agent]
 
         transport_options = {:headers => {}}
         transport_options[:headers].merge!(setup_basic_auth(user, password))
         transport_options[:headers].merge!(setup_api_key(api_key))
+        transport_options[:headers].merge!({ 'user-agent' => "#{user_agent}" })
 
         logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
         transport_options[:proxy] = proxy.to_s if proxy && !proxy.eql?('')

data/lib/logstash/filters/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb

@@ -20,7 +20,18 @@ if es_client_version >= Gem::Version.new('7.2') && es_client_version < Gem::Vers
             def apply_headers(request_options, options)
               headers = (options && options[:headers]) || {}
               headers[CONTENT_TYPE_STR] = find_value(headers, CONTENT_TYPE_REGEX) || DEFAULT_CONTENT_TYPE
-              headers[USER_AGENT_STR] = find_value(headers, USER_AGENT_REGEX) || user_agent_header
+
+              # this code is necessary to grab the correct user-agent header
+              # when this method is invoked with apply_headers(@request_options, options)
+              # from https://github.com/elastic/elasticsearch-ruby/blob/v7.14.0/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb#L113-L114
+              transport_user_agent = nil
+              if (options && options[:transport_options] && options[:transport_options][:headers])
+                transport_headers = {}
+                transport_headers = options[:transport_options][:headers]
+                transport_user_agent = find_value(transport_headers, USER_AGENT_REGEX)
+              end
+
+              headers[USER_AGENT_STR] = transport_user_agent || find_value(headers, USER_AGENT_REGEX) || user_agent_header
               headers[ACCEPT_ENCODING] = GZIP if use_compression?
               (request_options[:headers] ||= {}).merge!(headers) # this line was changed
             end

data/lib/logstash/filters/elasticsearch.rb

@@ -176,6 +176,19 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     end
   end # def filter
 
+  # public only to be reuse in testing
+  def prepare_user_agent
+    os_name = java.lang.System.getProperty('os.name')
+    os_version = java.lang.System.getProperty('os.version')
+    os_arch = java.lang.System.getProperty('os.arch')
+    jvm_vendor = java.lang.System.getProperty('java.vendor')
+    jvm_version = java.lang.System.getProperty('java.version')
+
+    plugin_version = Gem.loaded_specs['logstash-filter-elasticsearch'].version
+    # example: logstash/7.14.1 (OS=Linux-5.4.0-84-generic-amd64; JVM=AdoptOpenJDK-11.0.11) logstash-output-elasticsearch/11.0.1
+    "logstash/#{LOGSTASH_VERSION} (OS=#{os_name}-#{os_version}-#{os_arch}; JVM=#{jvm_vendor}-#{jvm_version}) logstash-#{@plugin_type}-#{config_name}/#{plugin_version}"
+  end
+
   private
 
   def client_options
@@ -192,7 +205,11 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   def new_client
     # NOTE: could pass cloud-id/cloud-auth to client but than we would need to be stricter on ES version requirement
     # and also LS parsing might differ from ES client's parsing so for consistency we do not pass cloud options ...
-    LogStash::Filters::ElasticsearchClient.new(@logger, @hosts, client_options)
+    opts = client_options
+
+    opts[:user_agent] = prepare_user_agent
+
+    LogStash::Filters::ElasticsearchClient.new(@logger, @hosts, opts)
   end
 
   def get_client

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.9.5'
+  s.version         = '3.10.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -23,6 +23,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'elasticsearch', ">= 5.0.5" # LS >= 6.7 and < 7.14 all used version 5.0.5
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
+  s.add_development_dependency 'cabin', ['~> 0.6']
+  s.add_development_dependency 'webrick'
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/filters/elasticsearch_spec.rb

@@ -3,6 +3,9 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/plugin"
 require "logstash/filters/elasticsearch"
 require "logstash/json"
+require "cabin"
+require "webrick"
+require "uri"
 
 describe LogStash::Filters::Elasticsearch do
 
@@ -291,6 +294,112 @@ describe LogStash::Filters::Elasticsearch do
     end
   end
 
+  class StoppableServer
+
+    attr_reader :port
+
+    def initialize()
+      queue = Queue.new
+      @first_req_waiter = java.util.concurrent.CountDownLatch.new(1)
+      @first_request = nil
+
+      @t = java.lang.Thread.new(
+        proc do
+          begin
+            @server = WEBrick::HTTPServer.new :Port => 0, :DocumentRoot => ".",
+                     :Logger => Cabin::Channel.get, # silence WEBrick logging
+                     :StartCallback => Proc.new {
+                           queue.push("started")
+                         }
+            @port = @server.config[:Port]
+            @server.mount_proc '/' do |req, res|
+              res.body = '''
+              {
+                  "name": "ce7ccfb438e8",
+                  "cluster_name": "docker-cluster",
+                  "cluster_uuid": "DyR1hN03QvuCWXRy3jtb0g",
+                  "version": {
+                      "number": "7.13.1",
+                      "build_flavor": "default",
+                      "build_type": "docker",
+                      "build_hash": "9a7758028e4ea59bcab41c12004603c5a7dd84a9",
+                      "build_date": "2021-05-28T17:40:59.346932922Z",
+                      "build_snapshot": false,
+                      "lucene_version": "8.8.2",
+                      "minimum_wire_compatibility_version": "6.8.0",
+                      "minimum_index_compatibility_version": "6.0.0-beta1"
+                  },
+                  "tagline": "You Know, for Search"
+              }
+              '''
+              res.status = 200
+              res['Content-Type'] = 'application/json'
+              @first_request = req
+              @first_req_waiter.countDown()
+            end
+
+            @server.start
+          rescue => e
+            puts "Error in webserver thread #{e}"
+            # ignore
+          end
+        end
+      )
+      @t.daemon = true
+      @t.start
+      queue.pop # blocks until the server is up
+    end
+
+    def stop
+      @server.shutdown
+    end
+
+    def wait_receive_request
+      @first_req_waiter.await(2, java.util.concurrent.TimeUnit::SECONDS)
+      @first_request
+    end
+  end
+
+  describe "user-agent header" do
+    let!(:webserver) { StoppableServer.new } # webserver must be started before the call, so no lazy "let"
+
+    after :each do
+      webserver.stop
+    end
+
+    it "server should be started" do
+      require 'net/http'
+      response = nil
+      Net::HTTP.start('localhost', webserver.port) {|http|
+        response = http.request_get('/')
+      }
+      expect(response.code.to_i).to eq(200)
+    end
+
+    context "used by plugin" do
+      let(:config) do
+        {
+          "hosts" => ["localhost:#{webserver.port}"],
+          "query" => "response: 404",
+          "fields" => { "response" => "code" },
+          "docinfo_fields" => { "_index" => "es_index" },
+          "aggregation_fields" => { "bytes_avg" => "bytes_avg_ls_field" }
+        }
+      end
+      let(:plugin) { described_class.new(config) }
+      let(:event)  { LogStash::Event.new({}) }
+
+      it "client should sent the expect user-agent" do
+        plugin.register
+
+        request = webserver.wait_receive_request
+
+        expect(request.header['user-agent'].size).to eq(1)
+        expect(request.header['user-agent'][0]).to match(/logstash\/\d*\.\d*\.\d* \(OS=.*; JVM=.*\) logstash-filter-elasticsearch\/\d*\.\d*\.\d*/)
+      end
+    end
+  end
+
   describe "client" do
     let(:config) do
       {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.9.5
+  version: 3.10.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-20 00:00:00.000000000 Z
+date: 2021-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,6 +58,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  name: cabin
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: webrick
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: