logstash-integration-kafka 10.9.0-java → 10.10.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df9c89cdfcc2db6702409ec45ffb8d9f8f2b9274304889df1527e0697ccfcf95
-  data.tar.gz: e907ad2e277d27c8cdbe98ebd203af86a3505a85574d64ace4402607c075a69e
+  metadata.gz: 1033b1bc88694b441cc6b117c431792780093b27ead742684f4e903048ed54a5
+  data.tar.gz: e2a74687db7bba3ccc192a544142226dccb3b144bb11e5cfecd84dd4c26cbdf0
 SHA512:
-  metadata.gz: bac93eb957af9028a6efc6e31a66c94818ae61333fa738daa6606abdd325dbea206fea2cd905d891f2b341a7bc983f8eaf5a5471015ac9548bc902f941b4a0d9
-  data.tar.gz: 456739a2409ef5a42f007a23c8d0dbfceb3518e8e65b528c0f48266f2a219c2415a83a507fdab3ba028cbc5493d645c080ce191a0d39d7c1787557abded9a0e1
+  metadata.gz: b1e206f1bfbd4acbf6ca66d11f974c2116faf357da1212dfa740675dbce47ca1dea661fb0c185df687798d2f6a053dabb781d63d687cd981769d38de938c148a
+  data.tar.gz: ca5f79ea95cd3901b1f47b06e9465a4962c4aa585cb1441254a916fe38a5603c98b87178eb85883caac5c28215b0e42de39c1d810e742d79d33bb8c3df82501b

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 10.10.0
+
+  - Added config setting to enable 'zstd' compression in the Kafka output [#112](https://github.com/logstash-plugins/logstash-integration-kafka/pull/112)
+
 ## 10.9.0
   - Refactor: leverage codec when using schema registry [#106](https://github.com/logstash-plugins/logstash-integration-kafka/pull/106)
   

data/DEVELOPER.md

@@ -62,7 +62,7 @@ See http://kafka.apache.org/documentation.html#producerconfigs for details about
         kafka {
             topic_id => ... # string (required), The topic to produce the messages to
             broker_list => ... # string (optional), default: "localhost:9092", This is for bootstrapping and the producer will only use it for getting metadata
-            compression_codec => ... # string (optional), one of ["none", "gzip", "snappy"], default: "none"
+            compression_codec => ... # string (optional), one of ["none", "gzip", "snappy", "lz4", "zstd"], default: "none"
             compressed_topics => ... # string (optional), default: "", This parameter allows you to set whether compression should be turned on for particular
             request_required_acks => ... # number (optional), one of [-1, 0, 1], default: 0, This value controls when a produce request is considered completed
             serializer_class => ... # string, (optional) default: "kafka.serializer.StringEncoder", The serializer class for messages. The default encoder takes a byte[] and returns the same byte[]

data/docs/output-kafka.asciidoc

@@ -84,7 +84,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-buffer_memory>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-client_dns_lookup>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-client_id>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4"]`|No
+| <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No
 | <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-key_serializer>> |<<string,string>>|No
@@ -193,11 +193,11 @@ ip/port by allowing a logical application name to be included with the request
 [id="plugins-{type}s-{plugin}-compression_type"]
 ===== `compression_type` 
 
-  * Value can be any of: `none`, `gzip`, `snappy`, `lz4`
+  * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd`
   * Default value is `"none"`
 
 The compression type for all data generated by the producer.
-The default is none (i.e. no compression). Valid values are none, gzip, snappy, or lz4.
+The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd.
 
 [id="plugins-{type}s-{plugin}-jaas_path"]
 ===== `jaas_path` 

data/lib/logstash/outputs/kafka.rb

@@ -80,8 +80,8 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
   config :buffer_memory, :validate => :number, :default => 33_554_432 # (32M) Kafka default
   # The compression type for all data generated by the producer.
-  # The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
-  config :compression_type, :validate => ["none", "gzip", "snappy", "lz4"], :default => "none"
+  # The default is none (i.e. no compression). Valid values are none, gzip, snappy, lz4 or zstd.
+  config :compression_type, :validate => ["none", "gzip", "snappy", "lz4", "zstd"], :default => "none"
   # How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple 
   # IP addresses for a hostname, they will all be attempted to connect to before failing the 
   # connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.9.0'
+  s.version         = '10.10.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/integration/outputs/kafka_spec.rb

@@ -139,6 +139,25 @@ describe "outputs/kafka", :integration => true do
     # end
   end
 
+  context 'when using zstd compression' do
+    let(:test_topic) { 'logstash_integration_zstd_topic' }
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic, "compression_type" => "zstd"})
+      load_kafka_data(config)
+    end
+
+    # NOTE: depends on zstd-ruby gem which is using a C-extension
+    # it 'should have data integrity' do
+    #   messages = fetch_messages(test_topic)
+    #
+    #   expect(messages.size).to eq(num_events)
+    #   messages.each do |m|
+    #     expect(m.value).to eq(event.to_s)
+    #   end
+    # end
+  end
+
   context 'when using multi partition topic' do
     let(:num_events) { 100 } # ~ more than (batch.size) 16,384 bytes
     let(:test_topic) { 'logstash_integration_topic3' }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.9.0
+  version: 10.10.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-14 00:00:00.000000000 Z
+date: 2022-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement