logstash-integration-kafka 10.5.0-java → 10.5.1-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/docs/index.asciidoc +7 -2
- data/docs/input-kafka.asciidoc +20 -8
- data/docs/output-kafka.asciidoc +23 -10
- data/logstash-integration-kafka.gemspec +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 93e42aba43873cfb3c8165a31b184e8178f738787329a36bcde79b234d2f642a
|
|
4
|
+
data.tar.gz: 41cca0d58b15c3072d51d009399e01e6854dea4c1d0cc3791e52430365599d73
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b40a479faa7568a326b53fb721f692bdb4f66e0c87e3553d39b29456b3e8b61f4849d0c325c21c3a8247989c16467d4c72268673b0c7e7ee41de7af1d939a523
|
|
7
|
+
data.tar.gz: 1df3fe42154e9b035145a159fbca4605f3c82d8d61765af3242ec6a2ff571b9b9d2bd536e484dfba766ca353722976c0c81b1e8f70676f083cec0ab3b6983a4e
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
## 10.5.1
|
|
2
|
+
- [DOC]Replaced plugin_header file with plugin_header-integration file. [#46](https://github.com/logstash-plugins/logstash-integration-kafka/pull/46)
|
|
3
|
+
- [DOC]Update kafka client version across kafka integration docs [#47](https://github.com/logstash-plugins/logstash-integration-kafka/pull/47)
|
|
4
|
+
- [DOC]Replace hard-coded kafka client and doc path version numbers with attributes to simplify doc maintenance [#48](https://github.com/logstash-plugins/logstash-integration-kafka/pull/48)
|
|
5
|
+
|
|
1
6
|
## 10.5.0
|
|
2
7
|
- Changed: retry sending messages only for retriable exceptions [#27](https://github.com/logstash-plugins/logstash-integration-kafka/pull/29)
|
|
3
8
|
|
data/docs/index.asciidoc
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
:plugin: kafka
|
|
2
2
|
:type: integration
|
|
3
3
|
:no_codec:
|
|
4
|
+
:kafka_client: 2.4
|
|
4
5
|
|
|
5
6
|
///////////////////////////////////////////
|
|
6
7
|
START - GENERATED VARIABLES, DO NOT EDIT!
|
|
@@ -21,11 +22,15 @@ include::{include_path}/plugin_header.asciidoc[]
|
|
|
21
22
|
|
|
22
23
|
==== Description
|
|
23
24
|
|
|
24
|
-
The Kafka Integration Plugin provides integrated plugins for working with the
|
|
25
|
+
The Kafka Integration Plugin provides integrated plugins for working with the
|
|
26
|
+
https://kafka.apache.org/[Kafka] distributed streaming platform.
|
|
25
27
|
|
|
26
28
|
- {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin]
|
|
27
29
|
- {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin]
|
|
28
30
|
|
|
29
|
-
This plugin uses Kafka Client
|
|
31
|
+
This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official
|
|
32
|
+
https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka
|
|
33
|
+
compatibility reference]. If the linked compatibility wiki is not up-to-date,
|
|
34
|
+
please contact Kafka support/community to confirm compatibility.
|
|
30
35
|
|
|
31
36
|
:no_codec!:
|
data/docs/input-kafka.asciidoc
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
|
+
:integration: kafka
|
|
1
2
|
:plugin: kafka
|
|
2
3
|
:type: input
|
|
3
4
|
:default_codec: plain
|
|
5
|
+
:kafka_client: 2.4
|
|
6
|
+
:kafka_client_doc: 24
|
|
4
7
|
|
|
5
8
|
///////////////////////////////////////////
|
|
6
9
|
START - GENERATED VARIABLES, DO NOT EDIT!
|
|
@@ -17,15 +20,20 @@ END - GENERATED VARIABLES, DO NOT EDIT!
|
|
|
17
20
|
|
|
18
21
|
=== Kafka input plugin
|
|
19
22
|
|
|
20
|
-
include::{include_path}/plugin_header.asciidoc[]
|
|
23
|
+
include::{include_path}/plugin_header-integration.asciidoc[]
|
|
21
24
|
|
|
22
25
|
==== Description
|
|
23
26
|
|
|
24
27
|
This input will read events from a Kafka topic.
|
|
25
28
|
|
|
26
|
-
This plugin uses Kafka Client
|
|
29
|
+
This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the
|
|
30
|
+
official
|
|
31
|
+
https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka
|
|
32
|
+
compatibility reference]. If the linked compatibility wiki is not up-to-date,
|
|
33
|
+
please contact Kafka support/community to confirm compatibility.
|
|
27
34
|
|
|
28
|
-
If you require features not yet available in this plugin (including client
|
|
35
|
+
If you require features not yet available in this plugin (including client
|
|
36
|
+
version upgrades), please file an issue with details about what you need.
|
|
29
37
|
|
|
30
38
|
This input supports connecting to Kafka over:
|
|
31
39
|
|
|
@@ -46,9 +54,9 @@ the same `group_id`.
|
|
|
46
54
|
Ideally you should have as many threads as the number of partitions for a perfect balance --
|
|
47
55
|
more threads than partitions means that some threads will be idle
|
|
48
56
|
|
|
49
|
-
For more information see https://kafka.apache.org/
|
|
57
|
+
For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer
|
|
50
58
|
|
|
51
|
-
Kafka consumer configuration: https://kafka.apache.org/
|
|
59
|
+
Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs
|
|
52
60
|
|
|
53
61
|
==== Metadata fields
|
|
54
62
|
|
|
@@ -59,7 +67,11 @@ The following metadata from Kafka broker are added under the `[@metadata]` field
|
|
|
59
67
|
* `[@metadata][kafka][partition]`: Partition info for this message.
|
|
60
68
|
* `[@metadata][kafka][offset]`: Original record offset for this message.
|
|
61
69
|
* `[@metadata][kafka][key]`: Record key, if any.
|
|
62
|
-
* `[@metadata][kafka][timestamp]`: Timestamp in the Record.
|
|
70
|
+
* `[@metadata][kafka][timestamp]`: Timestamp in the Record.
|
|
71
|
+
Depending on your broker configuration, this can be
|
|
72
|
+
either when the record was created (default) or when it was received by the
|
|
73
|
+
broker. See more about property log.message.timestamp.type at
|
|
74
|
+
https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs
|
|
63
75
|
|
|
64
76
|
Metadata is only added to the event if the `decorate_events` option is set to true (it defaults to false).
|
|
65
77
|
|
|
@@ -73,7 +85,7 @@ This plugin supports these configuration options plus the <<plugins-{type}s-{plu
|
|
|
73
85
|
|
|
74
86
|
NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting,
|
|
75
87
|
and might change if Kafka's consumer defaults change.
|
|
76
|
-
See the https://kafka.apache.org/
|
|
88
|
+
See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details.
|
|
77
89
|
|
|
78
90
|
[cols="<,<,<",options="header",]
|
|
79
91
|
|=======================================================================
|
|
@@ -421,7 +433,7 @@ partition ownership amongst consumer instances, supported options are:
|
|
|
421
433
|
* `sticky`
|
|
422
434
|
* `cooperative_sticky`
|
|
423
435
|
|
|
424
|
-
These map to Kafka's corresponding https://kafka.apache.org/
|
|
436
|
+
These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`]
|
|
425
437
|
implementations.
|
|
426
438
|
|
|
427
439
|
[id="plugins-{type}s-{plugin}-poll_timeout_ms"]
|
data/docs/output-kafka.asciidoc
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
|
+
:integration: kafka
|
|
1
2
|
:plugin: kafka
|
|
2
3
|
:type: output
|
|
3
4
|
:default_codec: plain
|
|
5
|
+
:kafka_client: 2.4
|
|
6
|
+
:kafka_client_doc: 24
|
|
4
7
|
|
|
5
8
|
///////////////////////////////////////////
|
|
6
9
|
START - GENERATED VARIABLES, DO NOT EDIT!
|
|
@@ -17,15 +20,20 @@ END - GENERATED VARIABLES, DO NOT EDIT!
|
|
|
17
20
|
|
|
18
21
|
=== Kafka output plugin
|
|
19
22
|
|
|
20
|
-
include::{include_path}/plugin_header.asciidoc[]
|
|
23
|
+
include::{include_path}/plugin_header-integration.asciidoc[]
|
|
21
24
|
|
|
22
25
|
==== Description
|
|
23
26
|
|
|
24
27
|
Write events to a Kafka topic.
|
|
25
28
|
|
|
26
|
-
This plugin uses Kafka Client
|
|
29
|
+
This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the
|
|
30
|
+
official
|
|
31
|
+
https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka
|
|
32
|
+
compatibility reference]. If the linked compatibility wiki is not up-to-date,
|
|
33
|
+
please contact Kafka support/community to confirm compatibility.
|
|
27
34
|
|
|
28
|
-
If you require features not yet available in this plugin (including client
|
|
35
|
+
If you require features not yet available in this plugin (including client
|
|
36
|
+
version upgrades), please file an issue with details about what you need.
|
|
29
37
|
|
|
30
38
|
This output supports connecting to Kafka over:
|
|
31
39
|
|
|
@@ -36,9 +44,12 @@ By default security is disabled but can be turned on as needed.
|
|
|
36
44
|
|
|
37
45
|
The only required configuration is the topic_id.
|
|
38
46
|
|
|
39
|
-
The default codec is plain. Logstash will encode your events with not only the
|
|
47
|
+
The default codec is plain. Logstash will encode your events with not only the
|
|
48
|
+
message field but also with a timestamp and hostname.
|
|
49
|
+
|
|
50
|
+
If you want the full content of your events to be sent as json, you should set
|
|
51
|
+
the codec in the output configuration like this:
|
|
40
52
|
|
|
41
|
-
If you want the full content of your events to be sent as json, you should set the codec in the output configuration like this:
|
|
42
53
|
[source,ruby]
|
|
43
54
|
output {
|
|
44
55
|
kafka {
|
|
@@ -47,9 +58,11 @@ If you want the full content of your events to be sent as json, you should set t
|
|
|
47
58
|
}
|
|
48
59
|
}
|
|
49
60
|
|
|
50
|
-
For more information see
|
|
61
|
+
For more information see
|
|
62
|
+
https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer
|
|
51
63
|
|
|
52
|
-
Kafka producer configuration:
|
|
64
|
+
Kafka producer configuration:
|
|
65
|
+
https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs
|
|
53
66
|
|
|
54
67
|
[id="plugins-{type}s-{plugin}-options"]
|
|
55
68
|
==== Kafka Output Configuration Options
|
|
@@ -58,7 +71,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
|
58
71
|
|
|
59
72
|
NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting,
|
|
60
73
|
and might change if Kafka's producer defaults change.
|
|
61
|
-
See the https://kafka.apache.org/
|
|
74
|
+
See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details.
|
|
62
75
|
|
|
63
76
|
[cols="<,<,<",options="header",]
|
|
64
77
|
|=======================================================================
|
|
@@ -328,9 +341,9 @@ Kafka down, etc).
|
|
|
328
341
|
A value less than zero is a configuration error.
|
|
329
342
|
|
|
330
343
|
Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of
|
|
331
|
-
https://kafka.apache.org/
|
|
344
|
+
https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException]
|
|
332
345
|
and
|
|
333
|
-
https://kafka.apache.org/
|
|
346
|
+
https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException].
|
|
334
347
|
If producing a message throws any other exception, an error is logged and the message is dropped without retrying.
|
|
335
348
|
This prevents the Logstash pipeline from hanging indefinitely.
|
|
336
349
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'logstash-integration-kafka'
|
|
3
|
-
s.version = '10.5.
|
|
3
|
+
s.version = '10.5.1'
|
|
4
4
|
s.licenses = ['Apache-2.0']
|
|
5
5
|
s.summary = "Integration with Kafka - input and output plugins"
|
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-integration-kafka
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 10.5.
|
|
4
|
+
version: 10.5.1
|
|
5
5
|
platform: java
|
|
6
6
|
authors:
|
|
7
7
|
- Elastic
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2020-
|
|
11
|
+
date: 2020-08-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|