logstash-input-elasticsearch 4.12.0 → 4.12.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c8dcf80b30079fa7b38d9c8cb0a11127bc896af3c5efe9e10b92023a74d0034
-  data.tar.gz: b6f40095bd604f4770529c621397c86a569494a973aa32dc13d978f68d49ac44
+  metadata.gz: 4a4ed685c9f90446a47fc84cb4e52e585f164c5451c9dca3f7359b756077bbd4
+  data.tar.gz: 050b54e5c660a4ab436f57ab16ea20d6a64a6b8a8c063eaaca69cc2c77235bd4
 SHA512:
-  metadata.gz: 9ff54e3056fedf78f80fa77d11ccb16f267675ff091d0c8f09262368272ec9d063064459acfc5645764f08a066692215d2713e0b0e100c502ad5488873ba549a
-  data.tar.gz: 7d315f02bdce91ce2f74fa27c85f9962bcdf3548a97a1018d577c480777e370f503301d4482aa28a2a8b68d932f8c2fdde9a6e7f576b6210ded7e43cad692dce
+  metadata.gz: 902070db657622bd65bcba8c17470456366134aae23cd38fb70ec89c97686b74da591e6f47d34957f3e5753f0e029ad78b845dd6355d6d559b6c135e0c94d28e
+  data.tar.gz: 22391299a87f5ce0a3085b4be7886a7f7251c9e33bd7051996871ea7cc34cbd7fbc53a3d54d841722b319949c4422e0ab7e6ff7d04899db6255543363fc32f66

data/CHANGELOG.md

@@ -1,5 +1,14 @@
+## 4.12.3
+  - Fix: update Elasticsearch Ruby client to correctly customize 'user-agent' header[#171](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/171)
+
+## 4.12.2
+  - Fix: hosts => "es_host:port" regression [#168](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/168)
+
+## 4.12.1
+  - Fixed too_long_frame_exception by passing scroll_id in the body [#159](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/159)
+
 ## 4.12.0
-  - Feat: Update Elasticsearch client to 7.14.0 [#157](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/157)
+  - Feat: Update Elasticsearch client to 7.14.0 [#157](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/157)
 
 ## 4.11.0
   - Feat: add user-agent header passed to the Elasticsearch HTTP connection [#158](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/158)
@@ -11,7 +20,7 @@
   - Fixed SSL handshake hang indefinitely with proxy setup [#156](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/156)
 
 ## 4.9.2
-  - Fix: a regression (in LS 7.14.0) where due the elasticsearch client update (from 5.0.5 to 7.5.0) the `Authorization` 
+  - Fix: a regression (in LS 7.14.0) where due the elasticsearch client update (from 5.0.5 to 7.5.0) the `Authorization`
     header isn't passed, this leads to the plugin not being able to leverage `user`/`password` credentials set by the user.
     [#153](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/153)
 
@@ -55,7 +64,7 @@
   - Feat: Added support for cloud_id / cloud_auth configuration [#112](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/112)
 
 ## 4.4.0
-  - Changed Elasticsearch Client transport to use Manticore [#111](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/111) 
+  - Changed Elasticsearch Client transport to use Manticore [#111](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/111)
 
 ## 4.3.3
   - Loosen restrictions on Elasticsearch gem [#110](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/110)
@@ -94,7 +103,7 @@
 
 ## 4.0.2
   - Bump ES client to 5.0.2 to get content-type: json behavior
-  - Revert unneeded manticore change 
+  - Revert unneeded manticore change
 
 ## 4.0.1
   - Switch internal HTTP client to support TLSv1.2
@@ -107,7 +116,7 @@
     behavior
   - Improve documentation to show sort by \_doc, and how to add it to custom
     queries.
-    
+
 ## 3.0.2
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 

data/Gemfile

@@ -9,3 +9,6 @@ if Dir.exist?(logstash_path) && use_logstash_source
   gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
 end
+
+gem 'manticore', ENV['MANTICORE_VERSION'] if ENV['MANTICORE_VERSION']
+gem 'elasticsearch', ENV['ELASTICSEARCH_VERSION'] if ENV['ELASTICSEARCH_VERSION']

data/lib/logstash/inputs/elasticsearch.rb

@@ -347,7 +347,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   end
 
   def clear_scroll(scroll_id)
-    @client.clear_scroll(scroll_id: scroll_id) if scroll_id
+    @client.clear_scroll(:body => { :scroll_id => scroll_id }) if scroll_id
   rescue => e
     # ignore & log any clear_scroll errors
     logger.warn("Ignoring clear_scroll exception", message: e.message, exception: e.class)
@@ -386,13 +386,13 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
   def setup_hosts
     @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
-    if @ssl
-      @hosts.map do |h|
-        host, port = h.split(":")
-        { :host => host, :scheme => 'https', :port => port }
+    @hosts.map do |h|
+      if h.start_with?('http:', 'https:')
+        h
+      else
+        host, port = h.split(':')
+        { host: host, port: port, scheme: (@ssl ? 'https' : 'http') }
       end
-    else
-      @hosts
     end
   end
 

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.12.0'
+  s.version         = '4.12.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -25,7 +25,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0'
   s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
 
-  s.add_runtime_dependency 'elasticsearch', '>= 7.14.0' # LS >= 6.7 and < 7.14 all used version 5.0.5
+  s.add_runtime_dependency 'elasticsearch', '>= 7.17.1'
 
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'
@@ -33,7 +33,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
 
   s.add_development_dependency 'logstash-codec-plain'
-  s.add_development_dependency 'faraday', "~> 1"
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'timecop'
   s.add_development_dependency 'cabin', ['~> 0.6']

data/spec/es_helper.rb

@@ -7,25 +7,26 @@ module ESHelper
     end
   end
 
-  def self.get_client(options)
-    require 'elasticsearch/transport/transport/http/faraday' # supports user/password options
-    host, port = get_host_port.split(':')
-    host_opts = { host: host, port: port, scheme: 'http' }
-    ssl_opts = {}
-
-    if options[:ca_file]
-      ssl_opts = { ca_file: options[:ca_file], version: 'TLSv1.2', verify: false }
-      host_opts[:scheme] = 'https'
+  def self.curl_and_get_json_response(url, method: :get, args: nil); require 'open3'
+    cmd = "curl -s -v --show-error #{args} -X #{method.to_s.upcase} -k #{url}"
+    begin
+      out, err, status = Open3.capture3(cmd)
+    rescue Errno::ENOENT
+      fail "curl not available, make sure curl binary is installed and available on $PATH"
     end
 
-    if options[:user] && options[:password]
-      host_opts[:user] = options[:user]
-      host_opts[:password] = options[:password]
-    end
+    if status.success?
+      http_status = err.match(/< HTTP\/1.1 (.*?)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+      if http_status.strip[0].to_i > 2
+        warn out
+        fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
+      end
 
-    Elasticsearch::Client.new(hosts: [host_opts],
-                              transport_options: { ssl: ssl_opts },
-                              transport_class: Elasticsearch::Transport::Transport::HTTP::Faraday)
+      LogStash::Json.load(out)
+    else
+      warn out
+      fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
+    end
   end
 
   def self.doc_type

data/spec/inputs/elasticsearch_spec.rb

@@ -47,7 +47,6 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
   end
 
   it_behaves_like "an interruptible input plugin" do
-    let(:esclient) { double("elasticsearch-client") }
     let(:config) do
       {
         "schedule" => "* * * * * UTC"
@@ -55,7 +54,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
 
     before :each do
-      allow(Elasticsearch::Client).to receive(:new).and_return(esclient)
+      @esclient = double("elasticsearch-client")
+      allow(Elasticsearch::Client).to receive(:new).and_return(@esclient)
       hit = {
         "_index" => "logstash-2014.10.12",
         "_type" => "logs",
@@ -63,10 +63,10 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         "_score" => 1.0,
         "_source" => { "message" => ["ohayo"] }
       }
-      allow(esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
-      allow(esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
-      allow(esclient).to receive(:clear_scroll).and_return(nil)
-      allow(esclient).to receive(:ping)
+      allow(@esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
+      allow(@esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
+      allow(@esclient).to receive(:clear_scroll).and_return(nil)
+      allow(@esclient).to receive(:ping)
     end
   end
 
@@ -933,7 +933,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       {
         "hosts" => ["localhost"],
         "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
-        "schedule" => "* * * * * UTC"
+        "schedule" => "* * * * * * UTC" # every second
       }
     end
 
@@ -942,21 +942,14 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
 
     it "should properly schedule" do
-      Timecop.travel(Time.new(2000))
-      Timecop.scale(60)
-      runner = Thread.new do
-        expect(plugin).to receive(:do_run) {
-          queue << LogStash::Event.new({})
-        }.at_least(:twice)
-
-        plugin.run(queue)
-      end
-      sleep 3
+      expect(plugin).to receive(:do_run) {
+        queue << LogStash::Event.new({})
+      }.at_least(:twice)
+      runner = Thread.start { plugin.run(queue) }
+      sleep 3.0
       plugin.stop
-      runner.kill
       runner.join
-      expect(queue.size).to eq(2)
-      Timecop.return
+      expect(queue.size).to be >= 2
     end
 
   end

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -6,29 +6,46 @@ require_relative "../../../spec/es_helper"
 
 describe LogStash::Inputs::Elasticsearch do
 
-  let(:config)   { { 'hosts' => [ESHelper.get_host_port],
+  SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
+
+  let(:config)   { { 'hosts' => ["http#{SECURE_INTEGRATION ? 's' : nil}://#{ESHelper.get_host_port}"],
                      'index' => 'logs',
                      'query' => '{ "query": { "match": { "message": "Not found"} }}' } }
+
   let(:plugin) { described_class.new(config) }
   let(:event)  { LogStash::Event.new({}) }
   let(:client_options) { Hash.new }
 
+  let(:user) { ENV['ELASTIC_USER'] || 'simpleuser' }
+  let(:password) { ENV['ELASTIC_PASSWORD'] || 'abc123' }
+  let(:ca_file) { "spec/fixtures/test_certs/ca.crt" }
+
+  let(:es_url) do
+    es_url = ESHelper.get_host_port
+    SECURE_INTEGRATION ? "https://#{es_url}" : "http://#{es_url}"
+  end
+
+  let(:curl_args) do
+    config['user'] ? "-u #{config['user']}:#{config['password']}" : ''
+  end
+
   before(:each) do
-    @es = ESHelper.get_client(client_options)
     # Delete all templates first.
     # Clean ES of data before we start.
-    @es.indices.delete_template(:name => "*")
+    ESHelper.curl_and_get_json_response "#{es_url}/_index_template/*", method: 'DELETE', args: curl_args
     # This can fail if there are no indexes, ignore failure.
-    @es.indices.delete(:index => "*") rescue nil
+    ESHelper.curl_and_get_json_response( "#{es_url}/_index/*", method: 'DELETE', args: curl_args) rescue nil
+    doc_args = "#{curl_args} -H 'Content-Type: application/json' -d '{\"response\": 404, \"message\":\"Not Found\"}'"
     10.times do
-      ESHelper.index_doc(@es, :index => 'logs', :body => { :response => 404, :message=> 'Not Found'})
+      ESHelper.curl_and_get_json_response "#{es_url}/logs/_doc", method: 'POST', args: doc_args
     end
-    @es.indices.refresh
+    ESHelper.curl_and_get_json_response "#{es_url}/_refresh", method: 'POST', args: curl_args
   end
 
   after(:each) do
-    @es.indices.delete_template(:name => "*")
-    @es.indices.delete(:index => "*") rescue nil
+    ESHelper.curl_and_get_json_response "#{es_url}/_index_template/*", method: 'DELETE', args: curl_args
+    # This can fail if there are no indexes, ignore failure.
+    ESHelper.curl_and_get_json_response( "#{es_url}/_index/*", method: 'DELETE', args: curl_args) rescue nil
   end
 
   shared_examples 'an elasticsearch index plugin' do
@@ -45,7 +62,7 @@ describe LogStash::Inputs::Elasticsearch do
     end
   end
 
-  describe 'against an unsecured elasticsearch', :integration => true do
+  describe 'against an unsecured elasticsearch', integration: true do
     before(:each) do
       plugin.register
     end
@@ -53,10 +70,7 @@ describe LogStash::Inputs::Elasticsearch do
     it_behaves_like 'an elasticsearch index plugin'
   end
 
-  describe 'against a secured elasticsearch', :secure_integration => true do
-    let(:user) { ENV['ELASTIC_USER'] || 'simpleuser' }
-    let(:password) { ENV['ELASTIC_PASSWORD'] || 'abc123' }
-    let(:ca_file) { "spec/fixtures/test_certs/ca.crt" }
+  describe 'against a secured elasticsearch', secure_integration: true do
 
     let(:client_options) { { :ca_file => ca_file, :user => user, :password => password } }
 
@@ -78,4 +92,28 @@ describe LogStash::Inputs::Elasticsearch do
     end
 
   end
+
+  context 'setting host:port', integration: true do
+
+    let(:config) do
+      super().merge "hosts" => [ESHelper.get_host_port]
+    end
+
+    it_behaves_like 'an elasticsearch index plugin'
+
+  end
+
+  context 'setting host:port (and ssl)', secure_integration: true do
+
+    let(:client_options) { { :ca_file => ca_file, :user => user, :password => password } }
+
+    let(:config) do
+      config = super().merge "hosts" => [ESHelper.get_host_port]
+      config.merge('user' => user, 'password' => password, 'ssl' => true, 'ca_file' => ca_file)
+    end
+
+    it_behaves_like 'an elasticsearch index plugin'
+
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.12.0
+  version: 4.12.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-11 00:00:00.000000000 Z
+date: 2022-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: 7.17.1
   name: elasticsearch
   prerelease: false
   type: :runtime
@@ -85,7 +85,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: 7.17.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -156,20 +156,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1'
-  name: faraday
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: