logstash-input-elasticsearch 4.12.2 → 4.12.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: caea2fedebb629d87471f67b2b03f43186e7e2b02e854e3f6df7154a64316f16
-  data.tar.gz: 1d75f76b97d6d8ea481637399b1ea82668f5fb47ea52a2f1a51034594b55375f
+  metadata.gz: 4a4ed685c9f90446a47fc84cb4e52e585f164c5451c9dca3f7359b756077bbd4
+  data.tar.gz: 050b54e5c660a4ab436f57ab16ea20d6a64a6b8a8c063eaaca69cc2c77235bd4
 SHA512:
-  metadata.gz: 07af682a46bc7aa7bd48b0d4c4f0a4baa9e8dd5a8848821863b67fe4b2ee5890655d76de1de782938be0b73bf3057228a94ccbf155b7fb9ecaa81077831362e5
-  data.tar.gz: 0031f21cc9d7e41dfdbf9de400b33e21a1b9abda7c9ca2159fb21623ff5cf21258b99670d22090eda39ad42141408166c10317874ea6682e4e3b756e031355a1
+  metadata.gz: 902070db657622bd65bcba8c17470456366134aae23cd38fb70ec89c97686b74da591e6f47d34957f3e5753f0e029ad78b845dd6355d6d559b6c135e0c94d28e
+  data.tar.gz: 22391299a87f5ce0a3085b4be7886a7f7251c9e33bd7051996871ea7cc34cbd7fbc53a3d54d841722b319949c4422e0ab7e6ff7d04899db6255543363fc32f66

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.12.3
+  - Fix: update Elasticsearch Ruby client to correctly customize 'user-agent' header[#171](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/171)
+
 ## 4.12.2
   - Fix: hosts => "es_host:port" regression [#168](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/168)
 
@@ -17,7 +20,7 @@
   - Fixed SSL handshake hang indefinitely with proxy setup [#156](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/156)
 
 ## 4.9.2
-  - Fix: a regression (in LS 7.14.0) where due the elasticsearch client update (from 5.0.5 to 7.5.0) the `Authorization` 
+  - Fix: a regression (in LS 7.14.0) where due the elasticsearch client update (from 5.0.5 to 7.5.0) the `Authorization`
     header isn't passed, this leads to the plugin not being able to leverage `user`/`password` credentials set by the user.
     [#153](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/153)
 
@@ -61,7 +64,7 @@
   - Feat: Added support for cloud_id / cloud_auth configuration [#112](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/112)
 
 ## 4.4.0
-  - Changed Elasticsearch Client transport to use Manticore [#111](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/111) 
+  - Changed Elasticsearch Client transport to use Manticore [#111](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/111)
 
 ## 4.3.3
   - Loosen restrictions on Elasticsearch gem [#110](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/110)
@@ -100,7 +103,7 @@
 
 ## 4.0.2
   - Bump ES client to 5.0.2 to get content-type: json behavior
-  - Revert unneeded manticore change 
+  - Revert unneeded manticore change
 
 ## 4.0.1
   - Switch internal HTTP client to support TLSv1.2
@@ -113,7 +116,7 @@
     behavior
   - Improve documentation to show sort by \_doc, and how to add it to custom
     queries.
-    
+
 ## 3.0.2
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.12.2'
+  s.version         = '4.12.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -25,7 +25,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0'
   s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
 
-  s.add_runtime_dependency 'elasticsearch', '>= 7.14.0' # LS >= 6.7 and < 7.14 all used version 5.0.5
+  s.add_runtime_dependency 'elasticsearch', '>= 7.17.1'
 
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'
@@ -33,7 +33,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
 
   s.add_development_dependency 'logstash-codec-plain'
-  s.add_development_dependency 'faraday', "~> 1"
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'timecop'
   s.add_development_dependency 'cabin', ['~> 0.6']

data/spec/es_helper.rb

@@ -7,25 +7,26 @@ module ESHelper
     end
   end
 
-  def self.get_client(options)
-    require 'elasticsearch/transport/transport/http/faraday' # supports user/password options
-    host, port = get_host_port.split(':')
-    host_opts = { host: host, port: port, scheme: 'http' }
-    ssl_opts = {}
-
-    if options[:ca_file]
-      ssl_opts = { ca_file: options[:ca_file], version: 'TLSv1.2', verify: false }
-      host_opts[:scheme] = 'https'
+  def self.curl_and_get_json_response(url, method: :get, args: nil); require 'open3'
+    cmd = "curl -s -v --show-error #{args} -X #{method.to_s.upcase} -k #{url}"
+    begin
+      out, err, status = Open3.capture3(cmd)
+    rescue Errno::ENOENT
+      fail "curl not available, make sure curl binary is installed and available on $PATH"
     end
 
-    if options[:user] && options[:password]
-      host_opts[:user] = options[:user]
-      host_opts[:password] = options[:password]
-    end
+    if status.success?
+      http_status = err.match(/< HTTP\/1.1 (.*?)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+      if http_status.strip[0].to_i > 2
+        warn out
+        fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
+      end
 
-    Elasticsearch::Client.new(hosts: [host_opts],
-                              transport_options: { ssl: ssl_opts },
-                              transport_class: Elasticsearch::Transport::Transport::HTTP::Faraday)
+      LogStash::Json.load(out)
+    else
+      warn out
+      fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
+    end
   end
 
   def self.doc_type

data/spec/inputs/elasticsearch_spec.rb

@@ -47,7 +47,6 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
   end
 
   it_behaves_like "an interruptible input plugin" do
-    let(:esclient) { double("elasticsearch-client") }
     let(:config) do
       {
         "schedule" => "* * * * * UTC"
@@ -55,7 +54,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
 
     before :each do
-      allow(Elasticsearch::Client).to receive(:new).and_return(esclient)
+      @esclient = double("elasticsearch-client")
+      allow(Elasticsearch::Client).to receive(:new).and_return(@esclient)
       hit = {
         "_index" => "logstash-2014.10.12",
         "_type" => "logs",
@@ -63,10 +63,10 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         "_score" => 1.0,
         "_source" => { "message" => ["ohayo"] }
       }
-      allow(esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
-      allow(esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
-      allow(esclient).to receive(:clear_scroll).and_return(nil)
-      allow(esclient).to receive(:ping)
+      allow(@esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
+      allow(@esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
+      allow(@esclient).to receive(:clear_scroll).and_return(nil)
+      allow(@esclient).to receive(:ping)
     end
   end
 
@@ -933,7 +933,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       {
         "hosts" => ["localhost"],
         "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
-        "schedule" => "* * * * * UTC"
+        "schedule" => "* * * * * * UTC" # every second
       }
     end
 
@@ -942,21 +942,14 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
 
     it "should properly schedule" do
-      Timecop.travel(Time.new(2000))
-      Timecop.scale(60)
-      runner = Thread.new do
-        expect(plugin).to receive(:do_run) {
-          queue << LogStash::Event.new({})
-        }.at_least(:twice)
-
-        plugin.run(queue)
-      end
-      sleep 3
+      expect(plugin).to receive(:do_run) {
+        queue << LogStash::Event.new({})
+      }.at_least(:twice)
+      runner = Thread.start { plugin.run(queue) }
+      sleep 3.0
       plugin.stop
-      runner.kill
       runner.join
-      expect(queue.size).to eq(2)
-      Timecop.return
+      expect(queue.size).to be >= 2
     end
 
   end

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -11,6 +11,7 @@ describe LogStash::Inputs::Elasticsearch do
   let(:config)   { { 'hosts' => ["http#{SECURE_INTEGRATION ? 's' : nil}://#{ESHelper.get_host_port}"],
                      'index' => 'logs',
                      'query' => '{ "query": { "match": { "message": "Not found"} }}' } }
+
   let(:plugin) { described_class.new(config) }
   let(:event)  { LogStash::Event.new({}) }
   let(:client_options) { Hash.new }
@@ -19,22 +20,32 @@ describe LogStash::Inputs::Elasticsearch do
   let(:password) { ENV['ELASTIC_PASSWORD'] || 'abc123' }
   let(:ca_file) { "spec/fixtures/test_certs/ca.crt" }
 
+  let(:es_url) do
+    es_url = ESHelper.get_host_port
+    SECURE_INTEGRATION ? "https://#{es_url}" : "http://#{es_url}"
+  end
+
+  let(:curl_args) do
+    config['user'] ? "-u #{config['user']}:#{config['password']}" : ''
+  end
+
   before(:each) do
-    @es = ESHelper.get_client(client_options)
     # Delete all templates first.
     # Clean ES of data before we start.
-    @es.indices.delete_template(:name => "*")
+    ESHelper.curl_and_get_json_response "#{es_url}/_index_template/*", method: 'DELETE', args: curl_args
     # This can fail if there are no indexes, ignore failure.
-    @es.indices.delete(:index => "*") rescue nil
+    ESHelper.curl_and_get_json_response( "#{es_url}/_index/*", method: 'DELETE', args: curl_args) rescue nil
+    doc_args = "#{curl_args} -H 'Content-Type: application/json' -d '{\"response\": 404, \"message\":\"Not Found\"}'"
     10.times do
-      ESHelper.index_doc(@es, :index => 'logs', :body => { :response => 404, :message=> 'Not Found'})
+      ESHelper.curl_and_get_json_response "#{es_url}/logs/_doc", method: 'POST', args: doc_args
     end
-    @es.indices.refresh
+    ESHelper.curl_and_get_json_response "#{es_url}/_refresh", method: 'POST', args: curl_args
   end
 
   after(:each) do
-    @es.indices.delete_template(:name => "*")
-    @es.indices.delete(:index => "*") rescue nil
+    ESHelper.curl_and_get_json_response "#{es_url}/_index_template/*", method: 'DELETE', args: curl_args
+    # This can fail if there are no indexes, ignore failure.
+    ESHelper.curl_and_get_json_response( "#{es_url}/_index/*", method: 'DELETE', args: curl_args) rescue nil
   end
 
   shared_examples 'an elasticsearch index plugin' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.12.2
+  version: 4.12.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-21 00:00:00.000000000 Z
+date: 2022-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: 7.17.1
   name: elasticsearch
   prerelease: false
   type: :runtime
@@ -85,7 +85,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: 7.17.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -156,20 +156,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1'
-  name: faraday
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: