logstash-integration-kafka 10.8.0-java → 10.10.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a322e757007ef9c740f48a93ebe97aa50511a29b17450219ea4f2d08a49934e1
-  data.tar.gz: 111a1d72591d14876f33dba65cfc29e6c9ae397397f295e3a94e5078b413eb3a
+  metadata.gz: 1033b1bc88694b441cc6b117c431792780093b27ead742684f4e903048ed54a5
+  data.tar.gz: e2a74687db7bba3ccc192a544142226dccb3b144bb11e5cfecd84dd4c26cbdf0
 SHA512:
-  metadata.gz: bac61cfb956bb284df97746df68afc90b12b1d04186eba0cf27fc68886858dad965aefcf46566dc8e27e8a484b524c7ce8d3185d24d9412798e06defc9f78ab8
-  data.tar.gz: f706aaad143f31fdd31ea47c5d1604f490f505f08329ecd07dff75201c51d1983701e79e2f74a52f484c860eb026a26880447e3b89a985dffeb7367728cab1ef
+  metadata.gz: b1e206f1bfbd4acbf6ca66d11f974c2116faf357da1212dfa740675dbce47ca1dea661fb0c185df687798d2f6a053dabb781d63d687cd981769d38de938c148a
+  data.tar.gz: ca5f79ea95cd3901b1f47b06e9465a4962c4aa585cb1441254a916fe38a5603c98b87178eb85883caac5c28215b0e42de39c1d810e742d79d33bb8c3df82501b

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+## 10.10.0
+
+  - Added config setting to enable 'zstd' compression in the Kafka output [#112](https://github.com/logstash-plugins/logstash-integration-kafka/pull/112)
+
+## 10.9.0
+  - Refactor: leverage codec when using schema registry [#106](https://github.com/logstash-plugins/logstash-integration-kafka/pull/106)
+  
+    Previously using `schema_registry_url` parsed the payload as JSON even if `codec => 'plain'` was set, this is no longer the case.  
+
+## 10.8.2
+  - [DOC] Updates description of `enable_auto_commit=false` to clarify that the commit happens after data is fetched AND written to the queue [#90](https://github.com/logstash-plugins/logstash-integration-kafka/pull/90)
+  - Fix: update to Gradle 7 [#104](https://github.com/logstash-plugins/logstash-integration-kafka/pull/104)
+  - [DOC] Clarify Kafka client does not support proxy [#103](https://github.com/logstash-plugins/logstash-integration-kafka/pull/103)
+
+## 10.8.1
+  - [DOC] Removed a setting recommendation that is no longer applicable for Kafka 2.0+ [#99](https://github.com/logstash-plugins/logstash-integration-kafka/pull/99)
+
 ## 10.8.0
   - Added config setting to enable schema registry validation to be skipped when an authentication scheme unsupported
     by the validator is used [#97](https://github.com/logstash-plugins/logstash-integration-kafka/pull/97)

data/DEVELOPER.md

@@ -62,7 +62,7 @@ See http://kafka.apache.org/documentation.html#producerconfigs for details about
         kafka {
             topic_id => ... # string (required), The topic to produce the messages to
             broker_list => ... # string (optional), default: "localhost:9092", This is for bootstrapping and the producer will only use it for getting metadata
-            compression_codec => ... # string (optional), one of ["none", "gzip", "snappy"], default: "none"
+            compression_codec => ... # string (optional), one of ["none", "gzip", "snappy", "lz4", "zstd"], default: "none"
             compressed_topics => ... # string (optional), default: "", This parameter allows you to set whether compression should be turned on for particular
             request_required_acks => ... # number (optional), one of [-1, 0, 1], default: 0, This value controls when a produce request is considered completed
             serializer_class => ... # string, (optional) default: "kafka.serializer.StringEncoder", The serializer class for messages. The default encoder takes a byte[] and returns the same byte[]

data/docs/input-kafka.asciidoc

@@ -42,6 +42,13 @@ This input supports connecting to Kafka over:
 
 By default security is disabled but can be turned on as needed.
 
+[NOTE]
+=======
+This plugin does not support using a proxy when communicating to the Kafka broker.  
+
+This plugin does support using a proxy when communicating to the Schema Registry using the <<plugins-{type}s-{plugin}-schema_registry_proxy>> option.
+=======
+
 The Logstash Kafka consumer handles group management and uses the default offset management
 strategy using Kafka topics.
 
@@ -276,7 +283,7 @@ which the consumption will begin.
 
 If true, periodically commit to Kafka the offsets of messages already returned by
 the consumer. If value is `false` however, the offset is committed every time the
-consumer fetches the data from the topic.
+consumer writes data fetched from the topic to the in-memory or persistent queue.
 
 [id="plugins-{type}s-{plugin}-exclude_internal_topics"]
 ===== `exclude_internal_topics` 
@@ -415,7 +422,6 @@ The maximum delay between invocations of poll() when using consumer group manage
 an upper bound on the amount of time that the consumer can be idle before fetching more records. 
 If poll() is not called before expiration of this timeout, then the consumer is considered failed and 
 the group will rebalance in order to reassign the partitions to another member.
-The value of the configuration `request_timeout_ms` must always be larger than `max_poll_interval_ms`. ???
 
 [id="plugins-{type}s-{plugin}-max_poll_records"]
 ===== `max_poll_records` 

data/docs/output-kafka.asciidoc

@@ -64,6 +64,8 @@ https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer
 Kafka producer configuration:
 https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs
 
+NOTE:  This plugin does not support using a proxy when communicating to the Kafka broker.
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Kafka Output Configuration Options
 
@@ -82,7 +84,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-buffer_memory>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-client_dns_lookup>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-client_id>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4"]`|No
+| <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No
 | <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-key_serializer>> |<<string,string>>|No
@@ -191,11 +193,11 @@ ip/port by allowing a logical application name to be included with the request
 [id="plugins-{type}s-{plugin}-compression_type"]
 ===== `compression_type` 
 
-  * Value can be any of: `none`, `gzip`, `snappy`, `lz4`
+  * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd`
   * Default value is `"none"`
 
 The compression type for all data generated by the producer.
-The default is none (i.e. no compression). Valid values are none, gzip, snappy, or lz4.
+The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd.
 
 [id="plugins-{type}s-{plugin}-jaas_path"]
 ===== `jaas_path` 

data/lib/logstash/inputs/kafka.rb

@@ -63,7 +63,12 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   config_name 'kafka'
 
-  default :codec, 'plain'
+  # default :codec, 'plain' or 'json' depending whether schema registry is used
+  #
+  # @override LogStash::Inputs::Base - removing the `:default => :plain`
+  config :codec, :validate => :codec
+  # NOTE: isn't necessary due the params['codec'] = ... done in #initialize
+  # having the `nil` default explicit makes the behavior more noticeable.
 
   # The frequency in milliseconds that the consumer offsets are committed to Kafka.
   config :auto_commit_interval_ms, :validate => :number, :default => 5000 # Kafka default
@@ -249,6 +254,15 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   attr_reader :metadata_mode
 
+  # @overload based on schema registry change the codec default
+  def initialize(params = {})
+    unless params.key?('codec')
+      params['codec'] = params.key?('schema_registry_url') ? 'json' : 'plain'
+    end
+
+    super(params)
+  end
+
   public
   def register
     @runner_threads = []
@@ -341,22 +355,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   def handle_record(record, codec_instance, queue)
     codec_instance.decode(record.value.to_s) do |event|
       decorate(event)
-      maybe_apply_schema(event, record)
       maybe_set_metadata(event, record)
       queue << event
     end
   end
 
-  def maybe_apply_schema(event, record)
-    if schema_registry_url
-      json = LogStash::Json.load(record.value.to_s)
-      json.each do |k, v|
-        event.set(k, v)
-      end
-      event.remove("message")
-    end
-  end
-
   def maybe_set_metadata(event, record)
     if @metadata_mode.include?(:record_props)
       event.set("[@metadata][kafka][topic]", record.topic)

data/lib/logstash/outputs/kafka.rb

@@ -80,8 +80,8 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
   config :buffer_memory, :validate => :number, :default => 33_554_432 # (32M) Kafka default
   # The compression type for all data generated by the producer.
-  # The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
-  config :compression_type, :validate => ["none", "gzip", "snappy", "lz4"], :default => "none"
+  # The default is none (i.e. no compression). Valid values are none, gzip, snappy, lz4 or zstd.
+  config :compression_type, :validate => ["none", "gzip", "snappy", "lz4", "zstd"], :default => "none"
   # How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple 
   # IP addresses for a hostname, they will all be attempted to connect to before failing the 
   # connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.8.0'
+  s.version         = '10.10.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -50,6 +50,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-codec-line'
   s.add_development_dependency 'rspec-wait'
   s.add_development_dependency 'digest-crc', '~> 0.5.1' # 0.6.0 started using a C-ext
   s.add_development_dependency 'ruby-kafka' # depends on digest-crc

data/spec/integration/inputs/kafka_spec.rb

@@ -16,38 +16,38 @@ describe "inputs/kafka", :integration => true do
   let(:group_id_5) {rand(36**8).to_s(36)}
   let(:group_id_6) {rand(36**8).to_s(36)}
   let(:plain_config) do
-    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1,
+    { 'topics' => ['logstash_integration_topic_plain'], 'group_id' => group_id_1,
       'auto_offset_reset' => 'earliest' }
   end
   let(:multi_consumer_config) do
     plain_config.merge({"group_id" => group_id_4, "client_id" => "spec", "consumer_threads" => 3})
   end
   let(:snappy_config) do
-    { 'topics' => ['logstash_integration_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1,
+    { 'topics' => ['logstash_integration_topic_snappy'], 'group_id' => group_id_1,
       'auto_offset_reset' => 'earliest' }
   end
   let(:lz4_config) do
-    { 'topics' => ['logstash_integration_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1,
+    { 'topics' => ['logstash_integration_topic_lz4'], 'group_id' => group_id_1,
       'auto_offset_reset' => 'earliest' }
   end
   let(:pattern_config) do
-    { 'topics_pattern' => 'logstash_integration_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain',
+    { 'topics_pattern' => 'logstash_integration_topic_.*', 'group_id' => group_id_2,
       'auto_offset_reset' => 'earliest' }
   end
   let(:decorate_config) do
-    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3,
+    { 'topics' => ['logstash_integration_topic_plain'], 'group_id' => group_id_3,
       'auto_offset_reset' => 'earliest', 'decorate_events' => 'true' }
   end
   let(:decorate_headers_config) do
-    { 'topics' => ['logstash_integration_topic_plain_with_headers'], 'codec' => 'plain', 'group_id' => group_id_3,
+    { 'topics' => ['logstash_integration_topic_plain_with_headers'], 'group_id' => group_id_3,
       'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
   end
   let(:decorate_bad_headers_config) do
-    { 'topics' => ['logstash_integration_topic_plain_with_headers_badly'], 'codec' => 'plain', 'group_id' => group_id_3,
+    { 'topics' => ['logstash_integration_topic_plain_with_headers_badly'], 'group_id' => group_id_3,
       'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
   end
   let(:manual_commit_config) do
-    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5,
+    { 'topics' => ['logstash_integration_topic_plain'], 'group_id' => group_id_5,
       'auto_offset_reset' => 'earliest', 'enable_auto_commit' => 'false' }
   end
   let(:timeout_seconds) { 30 }
@@ -352,10 +352,7 @@ describe "Deserializing with the schema registry", :integration => true do
 
     let(:base_config) do
       {
-          'topics' => [avro_topic_name],
-          'codec' => 'plain',
-          'group_id' => group_id_1,
-          'auto_offset_reset' => 'earliest'
+          'topics' => [avro_topic_name], 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'
       }
     end
 

data/spec/integration/outputs/kafka_spec.rb

@@ -139,6 +139,25 @@ describe "outputs/kafka", :integration => true do
     # end
   end
 
+  context 'when using zstd compression' do
+    let(:test_topic) { 'logstash_integration_zstd_topic' }
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic, "compression_type" => "zstd"})
+      load_kafka_data(config)
+    end
+
+    # NOTE: depends on zstd-ruby gem which is using a C-extension
+    # it 'should have data integrity' do
+    #   messages = fetch_messages(test_topic)
+    #
+    #   expect(messages.size).to eq(num_events)
+    #   messages.each do |m|
+    #     expect(m.value).to eq(event.to_s)
+    #   end
+    # end
+  end
+
   context 'when using multi partition topic' do
     let(:num_events) { 100 } # ~ more than (batch.size) 16,384 bytes
     let(:test_topic) { 'logstash_integration_topic3' }

data/spec/unit/inputs/kafka_spec.rb

@@ -177,7 +177,23 @@ describe LogStash::Inputs::Kafka do
     end
   end
 
-  describe "schema registry parameter verification" do
+  it 'uses plain codec by default' do
+    expect( subject.codec ).to respond_to :decode
+    expect( subject.codec.class ).to be LogStash::Codecs::Plain
+  end
+
+  context 'with codec option' do
+
+    let(:config) { super().merge 'codec' => 'line' }
+
+    it 'uses specified codec' do
+      expect( subject.codec ).to respond_to :decode
+      expect( subject.codec.class ).to be LogStash::Codecs::Line
+    end
+
+  end
+
+  describe "schema registry" do
     let(:base_config) do {
           'schema_registry_url' => 'http://localhost:8081',
           'topics' => ['logstash'],
@@ -186,7 +202,7 @@ describe LogStash::Inputs::Kafka do
     end
 
     context "schema_registry_url" do
-     let(:config) { base_config }
+      let(:config) { base_config }
 
       it "conflict with value_deserializer_class should fail" do
         config['value_deserializer_class'] = 'my.fantasy.Deserializer'
@@ -197,6 +213,11 @@ describe LogStash::Inputs::Kafka do
         config['topics_pattern'] = 'topic_.*'
         expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of topics_pattern/
       end
+
+      it 'switches default codec to json' do
+        expect( subject.codec ).to respond_to :decode
+        expect( subject.codec.class ).to be LogStash::Codecs::JSON
+      end
     end
 
     context 'when kerberos auth is used' do
@@ -204,9 +225,8 @@ describe LogStash::Inputs::Kafka do
         context "with #{protocol}" do
           ['auto', 'skip'].each do |vsr|
             context "when validata_schema_registry is #{vsr}" do
-              let(:config) { base_config.merge({'security_protocol' => protocol,
-                                                'schema_registry_validation' => vsr})
-              }
+              let(:config) { base_config.merge({'security_protocol' => protocol, 'schema_registry_validation' => vsr}) }
+
               it 'skips verification' do
                 expect(subject).not_to receive(:check_for_schema_registry_connectivity_and_subjects)
                 expect { subject.register }.not_to raise_error

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.8.0
+  version: 10.10.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-13 00:00:00.000000000 Z
+date: 2022-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -154,6 +154,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-line
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -280,8 +294,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Integration with Kafka - input and output plugins