logstash-input-elasticsearch 4.12.2 → 4.12.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -4
- data/logstash-input-elasticsearch.gemspec +2 -3
- data/spec/es_helper.rb +17 -16
- data/spec/inputs/elasticsearch_spec.rb +13 -20
- data/spec/inputs/integration/elasticsearch_spec.rb +18 -7
- metadata +4 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4a4ed685c9f90446a47fc84cb4e52e585f164c5451c9dca3f7359b756077bbd4
|
4
|
+
data.tar.gz: 050b54e5c660a4ab436f57ab16ea20d6a64a6b8a8c063eaaca69cc2c77235bd4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 902070db657622bd65bcba8c17470456366134aae23cd38fb70ec89c97686b74da591e6f47d34957f3e5753f0e029ad78b845dd6355d6d559b6c135e0c94d28e
|
7
|
+
data.tar.gz: 22391299a87f5ce0a3085b4be7886a7f7251c9e33bd7051996871ea7cc34cbd7fbc53a3d54d841722b319949c4422e0ab7e6ff7d04899db6255543363fc32f66
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 4.12.3
|
2
|
+
- Fix: update Elasticsearch Ruby client to correctly customize 'user-agent' header[#171](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/171)
|
3
|
+
|
1
4
|
## 4.12.2
|
2
5
|
- Fix: hosts => "es_host:port" regression [#168](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/168)
|
3
6
|
|
@@ -17,7 +20,7 @@
|
|
17
20
|
- Fixed SSL handshake hang indefinitely with proxy setup [#156](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/156)
|
18
21
|
|
19
22
|
## 4.9.2
|
20
|
-
- Fix: a regression (in LS 7.14.0) where due the elasticsearch client update (from 5.0.5 to 7.5.0) the `Authorization`
|
23
|
+
- Fix: a regression (in LS 7.14.0) where due the elasticsearch client update (from 5.0.5 to 7.5.0) the `Authorization`
|
21
24
|
header isn't passed, this leads to the plugin not being able to leverage `user`/`password` credentials set by the user.
|
22
25
|
[#153](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/153)
|
23
26
|
|
@@ -61,7 +64,7 @@
|
|
61
64
|
- Feat: Added support for cloud_id / cloud_auth configuration [#112](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/112)
|
62
65
|
|
63
66
|
## 4.4.0
|
64
|
-
- Changed Elasticsearch Client transport to use Manticore [#111](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/111)
|
67
|
+
- Changed Elasticsearch Client transport to use Manticore [#111](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/111)
|
65
68
|
|
66
69
|
## 4.3.3
|
67
70
|
- Loosen restrictions on Elasticsearch gem [#110](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/110)
|
@@ -100,7 +103,7 @@
|
|
100
103
|
|
101
104
|
## 4.0.2
|
102
105
|
- Bump ES client to 5.0.2 to get content-type: json behavior
|
103
|
-
- Revert unneeded manticore change
|
106
|
+
- Revert unneeded manticore change
|
104
107
|
|
105
108
|
## 4.0.1
|
106
109
|
- Switch internal HTTP client to support TLSv1.2
|
@@ -113,7 +116,7 @@
|
|
113
116
|
behavior
|
114
117
|
- Improve documentation to show sort by \_doc, and how to add it to custom
|
115
118
|
queries.
|
116
|
-
|
119
|
+
|
117
120
|
## 3.0.2
|
118
121
|
- Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
|
119
122
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-elasticsearch'
|
4
|
-
s.version = '4.12.
|
4
|
+
s.version = '4.12.3'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads query results from an Elasticsearch cluster"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -25,7 +25,7 @@ Gem::Specification.new do |s|
|
|
25
25
|
s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0'
|
26
26
|
s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
|
27
27
|
|
28
|
-
s.add_runtime_dependency 'elasticsearch', '>= 7.
|
28
|
+
s.add_runtime_dependency 'elasticsearch', '>= 7.17.1'
|
29
29
|
|
30
30
|
s.add_runtime_dependency 'tzinfo'
|
31
31
|
s.add_runtime_dependency 'tzinfo-data'
|
@@ -33,7 +33,6 @@ Gem::Specification.new do |s|
|
|
33
33
|
s.add_runtime_dependency 'manticore', ">= 0.7.1"
|
34
34
|
|
35
35
|
s.add_development_dependency 'logstash-codec-plain'
|
36
|
-
s.add_development_dependency 'faraday', "~> 1"
|
37
36
|
s.add_development_dependency 'logstash-devutils'
|
38
37
|
s.add_development_dependency 'timecop'
|
39
38
|
s.add_development_dependency 'cabin', ['~> 0.6']
|
data/spec/es_helper.rb
CHANGED
@@ -7,25 +7,26 @@ module ESHelper
|
|
7
7
|
end
|
8
8
|
end
|
9
9
|
|
10
|
-
def self.
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
if options[:ca_file]
|
17
|
-
ssl_opts = { ca_file: options[:ca_file], version: 'TLSv1.2', verify: false }
|
18
|
-
host_opts[:scheme] = 'https'
|
10
|
+
def self.curl_and_get_json_response(url, method: :get, args: nil); require 'open3'
|
11
|
+
cmd = "curl -s -v --show-error #{args} -X #{method.to_s.upcase} -k #{url}"
|
12
|
+
begin
|
13
|
+
out, err, status = Open3.capture3(cmd)
|
14
|
+
rescue Errno::ENOENT
|
15
|
+
fail "curl not available, make sure curl binary is installed and available on $PATH"
|
19
16
|
end
|
20
17
|
|
21
|
-
if
|
22
|
-
|
23
|
-
|
24
|
-
|
18
|
+
if status.success?
|
19
|
+
http_status = err.match(/< HTTP\/1.1 (.*?)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
|
20
|
+
if http_status.strip[0].to_i > 2
|
21
|
+
warn out
|
22
|
+
fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
|
23
|
+
end
|
25
24
|
|
26
|
-
|
27
|
-
|
28
|
-
|
25
|
+
LogStash::Json.load(out)
|
26
|
+
else
|
27
|
+
warn out
|
28
|
+
fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
|
29
|
+
end
|
29
30
|
end
|
30
31
|
|
31
32
|
def self.doc_type
|
@@ -47,7 +47,6 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
47
47
|
end
|
48
48
|
|
49
49
|
it_behaves_like "an interruptible input plugin" do
|
50
|
-
let(:esclient) { double("elasticsearch-client") }
|
51
50
|
let(:config) do
|
52
51
|
{
|
53
52
|
"schedule" => "* * * * * UTC"
|
@@ -55,7 +54,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
55
54
|
end
|
56
55
|
|
57
56
|
before :each do
|
58
|
-
|
57
|
+
@esclient = double("elasticsearch-client")
|
58
|
+
allow(Elasticsearch::Client).to receive(:new).and_return(@esclient)
|
59
59
|
hit = {
|
60
60
|
"_index" => "logstash-2014.10.12",
|
61
61
|
"_type" => "logs",
|
@@ -63,10 +63,10 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
63
63
|
"_score" => 1.0,
|
64
64
|
"_source" => { "message" => ["ohayo"] }
|
65
65
|
}
|
66
|
-
allow(esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
|
67
|
-
allow(esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
|
68
|
-
allow(esclient).to receive(:clear_scroll).and_return(nil)
|
69
|
-
allow(esclient).to receive(:ping)
|
66
|
+
allow(@esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
|
67
|
+
allow(@esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
|
68
|
+
allow(@esclient).to receive(:clear_scroll).and_return(nil)
|
69
|
+
allow(@esclient).to receive(:ping)
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
@@ -933,7 +933,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
933
933
|
{
|
934
934
|
"hosts" => ["localhost"],
|
935
935
|
"query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
|
936
|
-
"schedule" => "* * * * * UTC"
|
936
|
+
"schedule" => "* * * * * * UTC" # every second
|
937
937
|
}
|
938
938
|
end
|
939
939
|
|
@@ -942,21 +942,14 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
942
942
|
end
|
943
943
|
|
944
944
|
it "should properly schedule" do
|
945
|
-
|
946
|
-
|
947
|
-
|
948
|
-
|
949
|
-
|
950
|
-
}.at_least(:twice)
|
951
|
-
|
952
|
-
plugin.run(queue)
|
953
|
-
end
|
954
|
-
sleep 3
|
945
|
+
expect(plugin).to receive(:do_run) {
|
946
|
+
queue << LogStash::Event.new({})
|
947
|
+
}.at_least(:twice)
|
948
|
+
runner = Thread.start { plugin.run(queue) }
|
949
|
+
sleep 3.0
|
955
950
|
plugin.stop
|
956
|
-
runner.kill
|
957
951
|
runner.join
|
958
|
-
expect(queue.size).to
|
959
|
-
Timecop.return
|
952
|
+
expect(queue.size).to be >= 2
|
960
953
|
end
|
961
954
|
|
962
955
|
end
|
@@ -11,6 +11,7 @@ describe LogStash::Inputs::Elasticsearch do
|
|
11
11
|
let(:config) { { 'hosts' => ["http#{SECURE_INTEGRATION ? 's' : nil}://#{ESHelper.get_host_port}"],
|
12
12
|
'index' => 'logs',
|
13
13
|
'query' => '{ "query": { "match": { "message": "Not found"} }}' } }
|
14
|
+
|
14
15
|
let(:plugin) { described_class.new(config) }
|
15
16
|
let(:event) { LogStash::Event.new({}) }
|
16
17
|
let(:client_options) { Hash.new }
|
@@ -19,22 +20,32 @@ describe LogStash::Inputs::Elasticsearch do
|
|
19
20
|
let(:password) { ENV['ELASTIC_PASSWORD'] || 'abc123' }
|
20
21
|
let(:ca_file) { "spec/fixtures/test_certs/ca.crt" }
|
21
22
|
|
23
|
+
let(:es_url) do
|
24
|
+
es_url = ESHelper.get_host_port
|
25
|
+
SECURE_INTEGRATION ? "https://#{es_url}" : "http://#{es_url}"
|
26
|
+
end
|
27
|
+
|
28
|
+
let(:curl_args) do
|
29
|
+
config['user'] ? "-u #{config['user']}:#{config['password']}" : ''
|
30
|
+
end
|
31
|
+
|
22
32
|
before(:each) do
|
23
|
-
@es = ESHelper.get_client(client_options)
|
24
33
|
# Delete all templates first.
|
25
34
|
# Clean ES of data before we start.
|
26
|
-
|
35
|
+
ESHelper.curl_and_get_json_response "#{es_url}/_index_template/*", method: 'DELETE', args: curl_args
|
27
36
|
# This can fail if there are no indexes, ignore failure.
|
28
|
-
|
37
|
+
ESHelper.curl_and_get_json_response( "#{es_url}/_index/*", method: 'DELETE', args: curl_args) rescue nil
|
38
|
+
doc_args = "#{curl_args} -H 'Content-Type: application/json' -d '{\"response\": 404, \"message\":\"Not Found\"}'"
|
29
39
|
10.times do
|
30
|
-
ESHelper.
|
40
|
+
ESHelper.curl_and_get_json_response "#{es_url}/logs/_doc", method: 'POST', args: doc_args
|
31
41
|
end
|
32
|
-
|
42
|
+
ESHelper.curl_and_get_json_response "#{es_url}/_refresh", method: 'POST', args: curl_args
|
33
43
|
end
|
34
44
|
|
35
45
|
after(:each) do
|
36
|
-
|
37
|
-
|
46
|
+
ESHelper.curl_and_get_json_response "#{es_url}/_index_template/*", method: 'DELETE', args: curl_args
|
47
|
+
# This can fail if there are no indexes, ignore failure.
|
48
|
+
ESHelper.curl_and_get_json_response( "#{es_url}/_index/*", method: 'DELETE', args: curl_args) rescue nil
|
38
49
|
end
|
39
50
|
|
40
51
|
shared_examples 'an elasticsearch index plugin' do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.12.
|
4
|
+
version: 4.12.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-03-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -77,7 +77,7 @@ dependencies:
|
|
77
77
|
requirements:
|
78
78
|
- - ">="
|
79
79
|
- !ruby/object:Gem::Version
|
80
|
-
version: 7.
|
80
|
+
version: 7.17.1
|
81
81
|
name: elasticsearch
|
82
82
|
prerelease: false
|
83
83
|
type: :runtime
|
@@ -85,7 +85,7 @@ dependencies:
|
|
85
85
|
requirements:
|
86
86
|
- - ">="
|
87
87
|
- !ruby/object:Gem::Version
|
88
|
-
version: 7.
|
88
|
+
version: 7.17.1
|
89
89
|
- !ruby/object:Gem::Dependency
|
90
90
|
requirement: !ruby/object:Gem::Requirement
|
91
91
|
requirements:
|
@@ -156,20 +156,6 @@ dependencies:
|
|
156
156
|
- - ">="
|
157
157
|
- !ruby/object:Gem::Version
|
158
158
|
version: '0'
|
159
|
-
- !ruby/object:Gem::Dependency
|
160
|
-
requirement: !ruby/object:Gem::Requirement
|
161
|
-
requirements:
|
162
|
-
- - "~>"
|
163
|
-
- !ruby/object:Gem::Version
|
164
|
-
version: '1'
|
165
|
-
name: faraday
|
166
|
-
prerelease: false
|
167
|
-
type: :development
|
168
|
-
version_requirements: !ruby/object:Gem::Requirement
|
169
|
-
requirements:
|
170
|
-
- - "~>"
|
171
|
-
- !ruby/object:Gem::Version
|
172
|
-
version: '1'
|
173
159
|
- !ruby/object:Gem::Dependency
|
174
160
|
requirement: !ruby/object:Gem::Requirement
|
175
161
|
requirements:
|