logstash-output-elasticsearch 11.4.1-java → 11.4.2-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/docs/index.asciidoc +5 -0
- data/lib/logstash/outputs/elasticsearch/http_client.rb +3 -0
- data/lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-7x.json +1 -1
- data/lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-8x.json +1 -1
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/fixtures/test_certs/ca.crt +29 -30
- data/spec/fixtures/test_certs/renew.sh +11 -0
- data/spec/fixtures/test_certs/test.crt +18 -20
- data/spec/fixtures/test_certs/test.p12 +0 -0
- data/spec/integration/outputs/compressed_indexing_spec.rb +3 -1
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +8 -6
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a5aa752898bbe2a9417fc15ec7fb1e9a55a804a03c84cef30a936206d5b97005
|
4
|
+
data.tar.gz: 89ec0c8d8ff45e4ef4892c6b2a2a5de3f108d07443f925a897e058e79987fcba
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 633594b5c0010ca14c42231f8f1851bfdadd17819b61ea75f90d89c5db805d20d1a809ade265aedd432d80138965bdca18ec66819f387f69452f10d07a2ddb88
|
7
|
+
data.tar.gz: 9b4b8afaf80c0a9eebe7f7443c4fba5073c73823de87259641d373be62b9f52eb72e527d28d1dd4aca724abb487e13d1f49eff16f5ea0d528ab2ac7722522317
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 11.4.2
|
2
|
+
- Fixes an issue where events containing non-unicode strings could fail to serialize correctly when compression is enabled [#1169](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1169)
|
3
|
+
|
1
4
|
## 11.4.1
|
2
5
|
- Feat: upgrade manticore (http-client) library [#1063](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1063)
|
3
6
|
- the underlying changes include latest HttpClient (4.5.13)
|
data/docs/index.asciidoc
CHANGED
@@ -293,6 +293,11 @@ index level and `monitoring` permissions at cluster level. The `monitoring`
|
|
293
293
|
permission at cluster level is necessary to perform periodic connectivity
|
294
294
|
checks.
|
295
295
|
|
296
|
+
[id="plugins-{type}s-{plugin}-handling-non-utf-8"]
|
297
|
+
==== Handling non UTF-8 data
|
298
|
+
|
299
|
+
This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8.
|
300
|
+
When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`).
|
296
301
|
|
297
302
|
[id="plugins-{type}s-{plugin}-options"]
|
298
303
|
==== Elasticsearch Output Configuration Options
|
@@ -127,6 +127,9 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
127
127
|
action.map {|line| LogStash::Json.dump(line)}.join("\n") :
|
128
128
|
LogStash::Json.dump(action)
|
129
129
|
as_json << "\n"
|
130
|
+
|
131
|
+
as_json.scrub! # ensure generated JSON is valid UTF-8
|
132
|
+
|
130
133
|
if (stream_writer.pos + as_json.bytesize) > TARGET_BULK_BYTES && stream_writer.pos > 0
|
131
134
|
stream_writer.flush # ensure writer has sync'd buffers before reporting sizes
|
132
135
|
logger.debug("Sending partial bulk request for batch with one or more actions remaining.",
|
@@ -1,32 +1,31 @@
|
|
1
1
|
-----BEGIN CERTIFICATE-----
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
nIg9fQPvVw+fIHACZQ==
|
2
|
+
MIIFRTCCAy2gAwIBAgIBATANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
|
3
|
+
MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN
|
4
|
+
BgNVBAMMBlJvb3RDQTAeFw0yNDAzMTkyMjM3MTdaFw0yNTAzMTkyMjM3MTdaMEwx
|
5
|
+
CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD
|
6
|
+
VQQKDAVNeUxhYjEPMA0GA1UEAwwGUm9vdENBMIICIjANBgkqhkiG9w0BAQEFAAOC
|
7
|
+
Ag8AMIICCgKCAgEAy1MyoBa4fXtv1eo9rkvcc2qCdn3nz6C9w63tD+w4S9wNAmCT
|
8
|
+
Nn4bLCHl6vkkXaKiZg4eIPkmdxivhiZFAq5h8PoHVYjkW5C2EP86UDX9Eeq1tjbs
|
9
|
+
nfdJo7rqujyBqXu+MetcpCR59VhHB187oOqpuFXoviwyLLwXNDnlMymgzflxa6+g
|
10
|
+
AzG9JCoZnilhgqd81IaHMe+yx81LXG78vBvtWO7iM+Gn7jcGQbASKYjmSbuM0LWf
|
11
|
+
COIe3EOxj+z3cApr+8uS1cpQrmcDeOMk6EBtFNWds4CDEW3Rhtf3zFb9pSqxyAFR
|
12
|
+
Wz0n8zJNEzBUontWya2HU90lCSxQBK7MRKVI+XT10yNND4xfMkO3Qm9fxIgk+ZsG
|
13
|
+
eDvwxJoJSGk5mKKLWXaYF89Z6PHbQj9IwJQ2bNvCbbD0kPnQm/aJfiXiB7gfUIgr
|
14
|
+
X+itbWl1j3E8vS4piboAOwMLQHywaA0wFd4HpouiNjX8hPCaZ+l+T1z/JY98Luy0
|
15
|
+
eAJhEC/kx8YCya/T/JrlWtZAYEmQWJjLWmifqfFSMJuCfREAGiNGgZ2GFjdAOKmD
|
16
|
+
CZKAVnGvTrhEWvVu38yXcwmY9/nfmZLA92T+P/PDghlO+WbApbcXIwOHBgv/Nfsn
|
17
|
+
RzozMw8sR2LvdCPOavcQ0iuKvg7zrA3PzVhF9iSD1lfzIokr0sEBStgKMncCAwEA
|
18
|
+
AaMyMDAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUoVp0nHjq6mJ/UGuFhnSK
|
19
|
+
7yjbPyswDQYJKoZIhvcNAQELBQADggIBADeI0gRfzF5zyhCCmtlA44L1fo3BEYtm
|
20
|
+
0Deq2W6bsW9zakYT7fivCUbt0VtS0jzhAlbaLfksJk+Emg+kpsy/CdFr8nk9jlpN
|
21
|
+
XJMKwjQIVGYDDlJsP9VDw2m+HhUugEpx09QCREaCWCDtobE9bd42VloPd0u3VWZx
|
22
|
+
u5rSrZCy5wjFeq3dPEFPyDTfV6p1kXkwy9pdG3ww0rw0dqj5OqcGhLyG8b7q7LLP
|
23
|
+
aFVuwjnutrBJtDNOsvbD/dyybQVj3hF1IpVZLwoFQ3ji43/X3NJ5jdo/WrwwdwbH
|
24
|
+
NE6YxNlsIrql4sJFeHzNukTxcC5wHcOLtc6E6dhe7uK+5rRD75+odz85rPLsBTPt
|
25
|
+
zje8OWyxO1cGol7inpKUsO9ITFQdDC9iyRjvDRouwneT+XeVwW6BmAKg4DvXdLtQ
|
26
|
+
oQNLlMbbZWskvWoI4amXwSpzWmn4mRK3e4IPWFju64QRvkEExFynvC69jeih5Llh
|
27
|
+
JU3OjLobG4wKOQHB7w1UjiPJNsxGB0P1zug6ztTBLjsvJoxdJuIGkCsI7g12cKNR
|
28
|
+
PbXp8453klofFpMNGlCP8dns92Qu20pPvYJcWiC9k42nYu3FHDMbAfSrFU90RNS6
|
29
|
+
hRWkpXZOMIY1GdudgsxanDtJgFRbkPzwNkMcBQ1538f9U4MrJ9vPsuKx4bYbZ//8
|
30
|
+
pIe/z3b1V9zV
|
32
31
|
-----END CERTIFICATE-----
|
@@ -0,0 +1,11 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
|
3
|
+
set -e
|
4
|
+
cd "$(dirname "$0")"
|
5
|
+
|
6
|
+
openssl x509 -x509toreq -copy_extensions copyall -in ca.crt -signkey ca.key -out ca.csr
|
7
|
+
openssl x509 -req -copy_extensions copyall -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr
|
8
|
+
|
9
|
+
openssl x509 -x509toreq -copy_extensions copyall -in test.crt -signkey test.key -out test.csr
|
10
|
+
openssl x509 -req -copy_extensions copyall -days 365 -in test.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out test.crt && rm test.csr
|
11
|
+
openssl pkcs12 -export -inkey test.key -in test.crt -passout "pass:1234567890" -out test.p12
|
@@ -1,7 +1,7 @@
|
|
1
1
|
-----BEGIN CERTIFICATE-----
|
2
|
-
|
2
|
+
MIIF1jCCA76gAwIBAgIBATANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
|
3
3
|
MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN
|
4
|
-
|
4
|
+
BgNVBAMMBlJvb3RDQTAeFw0yNDAzMTkyMjM3MTdaFw0yNTAzMTkyMjM3MTdaMFMx
|
5
5
|
CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD
|
6
6
|
VQQKDAVNeUxhYjEWMBQGA1UEAwwNZWxhc3RpY3NlYXJjaDCCAiIwDQYJKoZIhvcN
|
7
7
|
AQEBBQADggIPADCCAgoCggIBAMYhP2zPOE3ke9naeK+cIPNV91htuoGGARs+mlY/
|
@@ -15,22 +15,20 @@ bFMKspGHnytQZF+a+mc5H33G9HiPP3jZE2JjrWlOay+j6ImylMgjcZmHAgaUe3ET
|
|
15
15
|
tyczoQ5/L5BNiyA2h+1TU8jWicNDtl1+CtOsgEVBBHA6p/IHhsHbNZWPrYtIO9mh
|
16
16
|
hiJw1R5yrITXnjZY0rObITwyt/e6Sc3YnoQfsSGaLJEG0aDc0RALAhgzj+RY8086
|
17
17
|
2RKOyfdw1sw1RmJKdCf+dOzhPyDpvauvCxrL8UZQTzcBs+qpxOWnZFRWeNsLwoDn
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
VTrvnAM1L3LwqtYQYfUWUHYZFYdvh8layA2ImNE7yx/9wIIkw/L1j9m71Upi6WKR
|
35
|
-
FKbYFqzgpWksa+zZ2RYYplUAxq0wYw==
|
18
|
+
6JXXAgMBAAGjgbswgbgwCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJ
|
19
|
+
YIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0
|
20
|
+
ZTAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwHQYDVR0OBBYE
|
21
|
+
FG+/PTIbDpPPZTFsEsAbLNUy1wxQMB8GA1UdIwQYMBaAFKFadJx46upif1BrhYZ0
|
22
|
+
iu8o2z8rMA0GCSqGSIb3DQEBCwUAA4ICAQB6Fptgev/wPHn36lamR9RCBpqvh/Xg
|
23
|
+
4nSnUfT1EVkEI7vUKkC/+/XeRTfnc14Yclo3uiYBY7QM953FhXOmlH0Rx1wwZSME
|
24
|
+
ZzyO+1TWNVR2QI7/MhpXdKpqHhNLNTYKFa602ZTA5NXa9XDJ8T3kjbuAKiNwcuu7
|
25
|
+
RzRK0o3KOWe0uohuI6bgcpgpKqcxrbtUkrPOWAn+XK7JQVSEV1kst4uDr1S+041c
|
26
|
+
T+NHFQarDXw5g/Y0pHnU42dPyDvT/RrcBbzbe7qylFTbjKro/uFqPAayHlLMnGZp
|
27
|
+
ZthGAPJJ0pDzBBxGtUmwOnat/HKWe2UN/A3Xqmy0Za7adhrYGj0IaZ0SNblNYDKy
|
28
|
+
OjWtfOwLQrD9NEIhBXBE3KS5uyXP8kZESnHtIN0b7SMyP7lUxTQ/Rh8U7S6QD9AI
|
29
|
+
gc+AfBx8UO+fdW3Z4VK+DU0NGRP+127w0yiEz1OtfS87dWBQq+kk2Hlc9TfFeG0b
|
30
|
+
rckM4v54X8uTlnNfPo+RYRkMAXB/fnx2vVIyxTxFPo4bAnTFA6vnIz8rrRO+Vmh3
|
31
|
+
oOclhE+ZemZ3BEC+Tgpvb4XD8pvDLnNOYr8tuDjN9jNNDofxplbsjI44wnb1v3HX
|
32
|
+
M8Xe3P1WfqpWZniRmtVi/CWjkPy2JeC6zW4dn7JdkfEFHGsHQZrEZyrVrtVFPRFo
|
33
|
+
hbgPGNWCxOhWzg==
|
36
34
|
-----END CERTIFICATE-----
|
Binary file
|
@@ -10,10 +10,12 @@ end
|
|
10
10
|
|
11
11
|
describe "indexing with http_compression turned on", :integration => true do
|
12
12
|
let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
|
13
|
+
let(:event_with_invalid_utf_8_bytes) { LogStash::Event.new("message" => "Message from spacecraft which contains \xAC invalid \xD7 byte sequences.", "type" => type) }
|
13
14
|
let(:index) { 10.times.collect { rand(10).to_s }.join("") }
|
14
15
|
let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
|
15
16
|
let(:event_count) { 10000 + rand(500) }
|
16
|
-
|
17
|
+
# mix the events with valid and invalid UTF-8 payloads
|
18
|
+
let(:events) { event_count.times.map { |i| i%3 == 0 ? event : event_with_invalid_utf_8_bytes }.to_a }
|
17
19
|
let(:config) {
|
18
20
|
{
|
19
21
|
"hosts" => get_host_port,
|
@@ -243,12 +243,14 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
|
|
243
243
|
end
|
244
244
|
end
|
245
245
|
|
246
|
-
context "with
|
247
|
-
let(:
|
248
|
-
let(:
|
246
|
+
context "with multiple messages" do
|
247
|
+
let(:message_head) { "Spacecraft message" }
|
248
|
+
let(:message_tail) { "byte sequence" }
|
249
|
+
let(:invalid_utf_8_message) { "contains invalid \xAC" }
|
249
250
|
let(:actions) { [
|
250
|
-
["index", {:_id=>nil, :_index=>"logstash"}, {"message"=>
|
251
|
-
["index", {:_id=>nil, :_index=>"logstash"}, {"message"=>
|
251
|
+
["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message_head}],
|
252
|
+
["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> invalid_utf_8_message}],
|
253
|
+
["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message_tail}],
|
252
254
|
]}
|
253
255
|
it "executes one bulk_send operation" do
|
254
256
|
allow(subject).to receive(:join_bulk_responses)
|
@@ -258,7 +260,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
|
|
258
260
|
|
259
261
|
context "if one exceeds TARGET_BULK_BYTES" do
|
260
262
|
let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
|
261
|
-
let(:
|
263
|
+
let(:message_head) { "a" * (target_bulk_bytes + 1) }
|
262
264
|
it "executes two bulk_send operations" do
|
263
265
|
allow(subject).to receive(:join_bulk_responses)
|
264
266
|
expect(subject).to receive(:bulk_send).twice
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 11.4.
|
4
|
+
version: 11.4.2
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-03-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -246,6 +246,7 @@ files:
|
|
246
246
|
- spec/fixtures/template-with-policy-es8x.json
|
247
247
|
- spec/fixtures/test_certs/ca.crt
|
248
248
|
- spec/fixtures/test_certs/ca.key
|
249
|
+
- spec/fixtures/test_certs/renew.sh
|
249
250
|
- spec/fixtures/test_certs/test.crt
|
250
251
|
- spec/fixtures/test_certs/test.key
|
251
252
|
- spec/fixtures/test_certs/test.p12
|
@@ -328,6 +329,7 @@ test_files:
|
|
328
329
|
- spec/fixtures/template-with-policy-es8x.json
|
329
330
|
- spec/fixtures/test_certs/ca.crt
|
330
331
|
- spec/fixtures/test_certs/ca.key
|
332
|
+
- spec/fixtures/test_certs/renew.sh
|
331
333
|
- spec/fixtures/test_certs/test.crt
|
332
334
|
- spec/fixtures/test_certs/test.key
|
333
335
|
- spec/fixtures/test_certs/test.p12
|