logstash-output-elasticsearch 11.4.1-java → 11.4.2-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/docs/index.asciidoc +5 -0
- data/lib/logstash/outputs/elasticsearch/http_client.rb +3 -0
- data/lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-7x.json +1 -1
- data/lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-8x.json +1 -1
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/fixtures/test_certs/ca.crt +29 -30
- data/spec/fixtures/test_certs/renew.sh +11 -0
- data/spec/fixtures/test_certs/test.crt +18 -20
- data/spec/fixtures/test_certs/test.p12 +0 -0
- data/spec/integration/outputs/compressed_indexing_spec.rb +3 -1
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +8 -6
- metadata +4 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a5aa752898bbe2a9417fc15ec7fb1e9a55a804a03c84cef30a936206d5b97005
|
|
4
|
+
data.tar.gz: 89ec0c8d8ff45e4ef4892c6b2a2a5de3f108d07443f925a897e058e79987fcba
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 633594b5c0010ca14c42231f8f1851bfdadd17819b61ea75f90d89c5db805d20d1a809ade265aedd432d80138965bdca18ec66819f387f69452f10d07a2ddb88
|
|
7
|
+
data.tar.gz: 9b4b8afaf80c0a9eebe7f7443c4fba5073c73823de87259641d373be62b9f52eb72e527d28d1dd4aca724abb487e13d1f49eff16f5ea0d528ab2ac7722522317
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
## 11.4.2
|
|
2
|
+
- Fixes an issue where events containing non-unicode strings could fail to serialize correctly when compression is enabled [#1169](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1169)
|
|
3
|
+
|
|
1
4
|
## 11.4.1
|
|
2
5
|
- Feat: upgrade manticore (http-client) library [#1063](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1063)
|
|
3
6
|
- the underlying changes include latest HttpClient (4.5.13)
|
data/docs/index.asciidoc
CHANGED
|
@@ -293,6 +293,11 @@ index level and `monitoring` permissions at cluster level. The `monitoring`
|
|
|
293
293
|
permission at cluster level is necessary to perform periodic connectivity
|
|
294
294
|
checks.
|
|
295
295
|
|
|
296
|
+
[id="plugins-{type}s-{plugin}-handling-non-utf-8"]
|
|
297
|
+
==== Handling non UTF-8 data
|
|
298
|
+
|
|
299
|
+
This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8.
|
|
300
|
+
When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`).
|
|
296
301
|
|
|
297
302
|
[id="plugins-{type}s-{plugin}-options"]
|
|
298
303
|
==== Elasticsearch Output Configuration Options
|
|
@@ -127,6 +127,9 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
|
127
127
|
action.map {|line| LogStash::Json.dump(line)}.join("\n") :
|
|
128
128
|
LogStash::Json.dump(action)
|
|
129
129
|
as_json << "\n"
|
|
130
|
+
|
|
131
|
+
as_json.scrub! # ensure generated JSON is valid UTF-8
|
|
132
|
+
|
|
130
133
|
if (stream_writer.pos + as_json.bytesize) > TARGET_BULK_BYTES && stream_writer.pos > 0
|
|
131
134
|
stream_writer.flush # ensure writer has sync'd buffers before reporting sizes
|
|
132
135
|
logger.debug("Sending partial bulk request for batch with one or more actions remaining.",
|
|
@@ -1,32 +1,31 @@
|
|
|
1
1
|
-----BEGIN CERTIFICATE-----
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
nIg9fQPvVw+fIHACZQ==
|
|
2
|
+
MIIFRTCCAy2gAwIBAgIBATANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
|
|
3
|
+
MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN
|
|
4
|
+
BgNVBAMMBlJvb3RDQTAeFw0yNDAzMTkyMjM3MTdaFw0yNTAzMTkyMjM3MTdaMEwx
|
|
5
|
+
CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD
|
|
6
|
+
VQQKDAVNeUxhYjEPMA0GA1UEAwwGUm9vdENBMIICIjANBgkqhkiG9w0BAQEFAAOC
|
|
7
|
+
Ag8AMIICCgKCAgEAy1MyoBa4fXtv1eo9rkvcc2qCdn3nz6C9w63tD+w4S9wNAmCT
|
|
8
|
+
Nn4bLCHl6vkkXaKiZg4eIPkmdxivhiZFAq5h8PoHVYjkW5C2EP86UDX9Eeq1tjbs
|
|
9
|
+
nfdJo7rqujyBqXu+MetcpCR59VhHB187oOqpuFXoviwyLLwXNDnlMymgzflxa6+g
|
|
10
|
+
AzG9JCoZnilhgqd81IaHMe+yx81LXG78vBvtWO7iM+Gn7jcGQbASKYjmSbuM0LWf
|
|
11
|
+
COIe3EOxj+z3cApr+8uS1cpQrmcDeOMk6EBtFNWds4CDEW3Rhtf3zFb9pSqxyAFR
|
|
12
|
+
Wz0n8zJNEzBUontWya2HU90lCSxQBK7MRKVI+XT10yNND4xfMkO3Qm9fxIgk+ZsG
|
|
13
|
+
eDvwxJoJSGk5mKKLWXaYF89Z6PHbQj9IwJQ2bNvCbbD0kPnQm/aJfiXiB7gfUIgr
|
|
14
|
+
X+itbWl1j3E8vS4piboAOwMLQHywaA0wFd4HpouiNjX8hPCaZ+l+T1z/JY98Luy0
|
|
15
|
+
eAJhEC/kx8YCya/T/JrlWtZAYEmQWJjLWmifqfFSMJuCfREAGiNGgZ2GFjdAOKmD
|
|
16
|
+
CZKAVnGvTrhEWvVu38yXcwmY9/nfmZLA92T+P/PDghlO+WbApbcXIwOHBgv/Nfsn
|
|
17
|
+
RzozMw8sR2LvdCPOavcQ0iuKvg7zrA3PzVhF9iSD1lfzIokr0sEBStgKMncCAwEA
|
|
18
|
+
AaMyMDAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUoVp0nHjq6mJ/UGuFhnSK
|
|
19
|
+
7yjbPyswDQYJKoZIhvcNAQELBQADggIBADeI0gRfzF5zyhCCmtlA44L1fo3BEYtm
|
|
20
|
+
0Deq2W6bsW9zakYT7fivCUbt0VtS0jzhAlbaLfksJk+Emg+kpsy/CdFr8nk9jlpN
|
|
21
|
+
XJMKwjQIVGYDDlJsP9VDw2m+HhUugEpx09QCREaCWCDtobE9bd42VloPd0u3VWZx
|
|
22
|
+
u5rSrZCy5wjFeq3dPEFPyDTfV6p1kXkwy9pdG3ww0rw0dqj5OqcGhLyG8b7q7LLP
|
|
23
|
+
aFVuwjnutrBJtDNOsvbD/dyybQVj3hF1IpVZLwoFQ3ji43/X3NJ5jdo/WrwwdwbH
|
|
24
|
+
NE6YxNlsIrql4sJFeHzNukTxcC5wHcOLtc6E6dhe7uK+5rRD75+odz85rPLsBTPt
|
|
25
|
+
zje8OWyxO1cGol7inpKUsO9ITFQdDC9iyRjvDRouwneT+XeVwW6BmAKg4DvXdLtQ
|
|
26
|
+
oQNLlMbbZWskvWoI4amXwSpzWmn4mRK3e4IPWFju64QRvkEExFynvC69jeih5Llh
|
|
27
|
+
JU3OjLobG4wKOQHB7w1UjiPJNsxGB0P1zug6ztTBLjsvJoxdJuIGkCsI7g12cKNR
|
|
28
|
+
PbXp8453klofFpMNGlCP8dns92Qu20pPvYJcWiC9k42nYu3FHDMbAfSrFU90RNS6
|
|
29
|
+
hRWkpXZOMIY1GdudgsxanDtJgFRbkPzwNkMcBQ1538f9U4MrJ9vPsuKx4bYbZ//8
|
|
30
|
+
pIe/z3b1V9zV
|
|
32
31
|
-----END CERTIFICATE-----
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
|
|
3
|
+
set -e
|
|
4
|
+
cd "$(dirname "$0")"
|
|
5
|
+
|
|
6
|
+
openssl x509 -x509toreq -copy_extensions copyall -in ca.crt -signkey ca.key -out ca.csr
|
|
7
|
+
openssl x509 -req -copy_extensions copyall -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr
|
|
8
|
+
|
|
9
|
+
openssl x509 -x509toreq -copy_extensions copyall -in test.crt -signkey test.key -out test.csr
|
|
10
|
+
openssl x509 -req -copy_extensions copyall -days 365 -in test.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out test.crt && rm test.csr
|
|
11
|
+
openssl pkcs12 -export -inkey test.key -in test.crt -passout "pass:1234567890" -out test.p12
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
-----BEGIN CERTIFICATE-----
|
|
2
|
-
|
|
2
|
+
MIIF1jCCA76gAwIBAgIBATANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
|
|
3
3
|
MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN
|
|
4
|
-
|
|
4
|
+
BgNVBAMMBlJvb3RDQTAeFw0yNDAzMTkyMjM3MTdaFw0yNTAzMTkyMjM3MTdaMFMx
|
|
5
5
|
CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD
|
|
6
6
|
VQQKDAVNeUxhYjEWMBQGA1UEAwwNZWxhc3RpY3NlYXJjaDCCAiIwDQYJKoZIhvcN
|
|
7
7
|
AQEBBQADggIPADCCAgoCggIBAMYhP2zPOE3ke9naeK+cIPNV91htuoGGARs+mlY/
|
|
@@ -15,22 +15,20 @@ bFMKspGHnytQZF+a+mc5H33G9HiPP3jZE2JjrWlOay+j6ImylMgjcZmHAgaUe3ET
|
|
|
15
15
|
tyczoQ5/L5BNiyA2h+1TU8jWicNDtl1+CtOsgEVBBHA6p/IHhsHbNZWPrYtIO9mh
|
|
16
16
|
hiJw1R5yrITXnjZY0rObITwyt/e6Sc3YnoQfsSGaLJEG0aDc0RALAhgzj+RY8086
|
|
17
17
|
2RKOyfdw1sw1RmJKdCf+dOzhPyDpvauvCxrL8UZQTzcBs+qpxOWnZFRWeNsLwoDn
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
VTrvnAM1L3LwqtYQYfUWUHYZFYdvh8layA2ImNE7yx/9wIIkw/L1j9m71Upi6WKR
|
|
35
|
-
FKbYFqzgpWksa+zZ2RYYplUAxq0wYw==
|
|
18
|
+
6JXXAgMBAAGjgbswgbgwCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJ
|
|
19
|
+
YIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0
|
|
20
|
+
ZTAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwHQYDVR0OBBYE
|
|
21
|
+
FG+/PTIbDpPPZTFsEsAbLNUy1wxQMB8GA1UdIwQYMBaAFKFadJx46upif1BrhYZ0
|
|
22
|
+
iu8o2z8rMA0GCSqGSIb3DQEBCwUAA4ICAQB6Fptgev/wPHn36lamR9RCBpqvh/Xg
|
|
23
|
+
4nSnUfT1EVkEI7vUKkC/+/XeRTfnc14Yclo3uiYBY7QM953FhXOmlH0Rx1wwZSME
|
|
24
|
+
ZzyO+1TWNVR2QI7/MhpXdKpqHhNLNTYKFa602ZTA5NXa9XDJ8T3kjbuAKiNwcuu7
|
|
25
|
+
RzRK0o3KOWe0uohuI6bgcpgpKqcxrbtUkrPOWAn+XK7JQVSEV1kst4uDr1S+041c
|
|
26
|
+
T+NHFQarDXw5g/Y0pHnU42dPyDvT/RrcBbzbe7qylFTbjKro/uFqPAayHlLMnGZp
|
|
27
|
+
ZthGAPJJ0pDzBBxGtUmwOnat/HKWe2UN/A3Xqmy0Za7adhrYGj0IaZ0SNblNYDKy
|
|
28
|
+
OjWtfOwLQrD9NEIhBXBE3KS5uyXP8kZESnHtIN0b7SMyP7lUxTQ/Rh8U7S6QD9AI
|
|
29
|
+
gc+AfBx8UO+fdW3Z4VK+DU0NGRP+127w0yiEz1OtfS87dWBQq+kk2Hlc9TfFeG0b
|
|
30
|
+
rckM4v54X8uTlnNfPo+RYRkMAXB/fnx2vVIyxTxFPo4bAnTFA6vnIz8rrRO+Vmh3
|
|
31
|
+
oOclhE+ZemZ3BEC+Tgpvb4XD8pvDLnNOYr8tuDjN9jNNDofxplbsjI44wnb1v3HX
|
|
32
|
+
M8Xe3P1WfqpWZniRmtVi/CWjkPy2JeC6zW4dn7JdkfEFHGsHQZrEZyrVrtVFPRFo
|
|
33
|
+
hbgPGNWCxOhWzg==
|
|
36
34
|
-----END CERTIFICATE-----
|
|
Binary file
|
|
@@ -10,10 +10,12 @@ end
|
|
|
10
10
|
|
|
11
11
|
describe "indexing with http_compression turned on", :integration => true do
|
|
12
12
|
let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
|
|
13
|
+
let(:event_with_invalid_utf_8_bytes) { LogStash::Event.new("message" => "Message from spacecraft which contains \xAC invalid \xD7 byte sequences.", "type" => type) }
|
|
13
14
|
let(:index) { 10.times.collect { rand(10).to_s }.join("") }
|
|
14
15
|
let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
|
|
15
16
|
let(:event_count) { 10000 + rand(500) }
|
|
16
|
-
|
|
17
|
+
# mix the events with valid and invalid UTF-8 payloads
|
|
18
|
+
let(:events) { event_count.times.map { |i| i%3 == 0 ? event : event_with_invalid_utf_8_bytes }.to_a }
|
|
17
19
|
let(:config) {
|
|
18
20
|
{
|
|
19
21
|
"hosts" => get_host_port,
|
|
@@ -243,12 +243,14 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
|
|
|
243
243
|
end
|
|
244
244
|
end
|
|
245
245
|
|
|
246
|
-
context "with
|
|
247
|
-
let(:
|
|
248
|
-
let(:
|
|
246
|
+
context "with multiple messages" do
|
|
247
|
+
let(:message_head) { "Spacecraft message" }
|
|
248
|
+
let(:message_tail) { "byte sequence" }
|
|
249
|
+
let(:invalid_utf_8_message) { "contains invalid \xAC" }
|
|
249
250
|
let(:actions) { [
|
|
250
|
-
["index", {:_id=>nil, :_index=>"logstash"}, {"message"=>
|
|
251
|
-
["index", {:_id=>nil, :_index=>"logstash"}, {"message"=>
|
|
251
|
+
["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message_head}],
|
|
252
|
+
["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> invalid_utf_8_message}],
|
|
253
|
+
["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message_tail}],
|
|
252
254
|
]}
|
|
253
255
|
it "executes one bulk_send operation" do
|
|
254
256
|
allow(subject).to receive(:join_bulk_responses)
|
|
@@ -258,7 +260,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
|
|
|
258
260
|
|
|
259
261
|
context "if one exceeds TARGET_BULK_BYTES" do
|
|
260
262
|
let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
|
|
261
|
-
let(:
|
|
263
|
+
let(:message_head) { "a" * (target_bulk_bytes + 1) }
|
|
262
264
|
it "executes two bulk_send operations" do
|
|
263
265
|
allow(subject).to receive(:join_bulk_responses)
|
|
264
266
|
expect(subject).to receive(:bulk_send).twice
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-output-elasticsearch
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 11.4.
|
|
4
|
+
version: 11.4.2
|
|
5
5
|
platform: java
|
|
6
6
|
authors:
|
|
7
7
|
- Elastic
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2024-03-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -246,6 +246,7 @@ files:
|
|
|
246
246
|
- spec/fixtures/template-with-policy-es8x.json
|
|
247
247
|
- spec/fixtures/test_certs/ca.crt
|
|
248
248
|
- spec/fixtures/test_certs/ca.key
|
|
249
|
+
- spec/fixtures/test_certs/renew.sh
|
|
249
250
|
- spec/fixtures/test_certs/test.crt
|
|
250
251
|
- spec/fixtures/test_certs/test.key
|
|
251
252
|
- spec/fixtures/test_certs/test.p12
|
|
@@ -328,6 +329,7 @@ test_files:
|
|
|
328
329
|
- spec/fixtures/template-with-policy-es8x.json
|
|
329
330
|
- spec/fixtures/test_certs/ca.crt
|
|
330
331
|
- spec/fixtures/test_certs/ca.key
|
|
332
|
+
- spec/fixtures/test_certs/renew.sh
|
|
331
333
|
- spec/fixtures/test_certs/test.crt
|
|
332
334
|
- spec/fixtures/test_certs/test.key
|
|
333
335
|
- spec/fixtures/test_certs/test.p12
|