logstash-integration-kafka 10.8.2-java → 10.9.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e11499f791fd2bd71102ddb6f8e52fcbd9791feecf4107c7ebd27dc9334cc34
-  data.tar.gz: 779cd381a6e53155f61457229df4fc827ceda02ef4e069d4eab0240dafa4ab83
+  metadata.gz: df9c89cdfcc2db6702409ec45ffb8d9f8f2b9274304889df1527e0697ccfcf95
+  data.tar.gz: e907ad2e277d27c8cdbe98ebd203af86a3505a85574d64ace4402607c075a69e
 SHA512:
-  metadata.gz: 5577048fe1adee0c28b012c0bf6eb0ff594b7be4760f7b7ebdf9072473fb0a1806e24997dd7b0659c21f205396a4d7eccb10bc4e789541c35fb7e32610562632
-  data.tar.gz: 7ee0e7428d63957a57e927a3f0408bcdddf94efad944c5cf0acbac3c1031be55b6751ad0290ea134e5201f10f9e26f2804c6d68150e925f651c9db27847bab5e
+  metadata.gz: bac93eb957af9028a6efc6e31a66c94818ae61333fa738daa6606abdd325dbea206fea2cd905d891f2b341a7bc983f8eaf5a5471015ac9548bc902f941b4a0d9
+  data.tar.gz: 456739a2409ef5a42f007a23c8d0dbfceb3518e8e65b528c0f48266f2a219c2415a83a507fdab3ba028cbc5493d645c080ce191a0d39d7c1787557abded9a0e1

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 10.9.0
+  - Refactor: leverage codec when using schema registry [#106](https://github.com/logstash-plugins/logstash-integration-kafka/pull/106)
+  
+    Previously using `schema_registry_url` parsed the payload as JSON even if `codec => 'plain'` was set, this is no longer the case.  
+
 ## 10.8.2
   - [DOC] Updates description of `enable_auto_commit=false` to clarify that the commit happens after data is fetched AND written to the queue [#90](https://github.com/logstash-plugins/logstash-integration-kafka/pull/90)
   - Fix: update to Gradle 7 [#104](https://github.com/logstash-plugins/logstash-integration-kafka/pull/104)

data/lib/logstash/inputs/kafka.rb

@@ -63,7 +63,12 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   config_name 'kafka'
 
-  default :codec, 'plain'
+  # default :codec, 'plain' or 'json' depending whether schema registry is used
+  #
+  # @override LogStash::Inputs::Base - removing the `:default => :plain`
+  config :codec, :validate => :codec
+  # NOTE: isn't necessary due the params['codec'] = ... done in #initialize
+  # having the `nil` default explicit makes the behavior more noticeable.
 
   # The frequency in milliseconds that the consumer offsets are committed to Kafka.
   config :auto_commit_interval_ms, :validate => :number, :default => 5000 # Kafka default
@@ -249,6 +254,15 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   attr_reader :metadata_mode
 
+  # @overload based on schema registry change the codec default
+  def initialize(params = {})
+    unless params.key?('codec')
+      params['codec'] = params.key?('schema_registry_url') ? 'json' : 'plain'
+    end
+
+    super(params)
+  end
+
   public
   def register
     @runner_threads = []
@@ -341,22 +355,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   def handle_record(record, codec_instance, queue)
     codec_instance.decode(record.value.to_s) do |event|
       decorate(event)
-      maybe_apply_schema(event, record)
       maybe_set_metadata(event, record)
       queue << event
     end
   end
 
-  def maybe_apply_schema(event, record)
-    if schema_registry_url
-      json = LogStash::Json.load(record.value.to_s)
-      json.each do |k, v|
-        event.set(k, v)
-      end
-      event.remove("message")
-    end
-  end
-
   def maybe_set_metadata(event, record)
     if @metadata_mode.include?(:record_props)
       event.set("[@metadata][kafka][topic]", record.topic)

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.8.2'
+  s.version         = '10.9.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -50,6 +50,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-codec-line'
   s.add_development_dependency 'rspec-wait'
   s.add_development_dependency 'digest-crc', '~> 0.5.1' # 0.6.0 started using a C-ext
   s.add_development_dependency 'ruby-kafka' # depends on digest-crc

data/spec/integration/inputs/kafka_spec.rb

@@ -16,38 +16,38 @@ describe "inputs/kafka", :integration => true do
   let(:group_id_5) {rand(36**8).to_s(36)}
   let(:group_id_6) {rand(36**8).to_s(36)}
   let(:plain_config) do
-    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1,
+    { 'topics' => ['logstash_integration_topic_plain'], 'group_id' => group_id_1,
       'auto_offset_reset' => 'earliest' }
   end
   let(:multi_consumer_config) do
     plain_config.merge({"group_id" => group_id_4, "client_id" => "spec", "consumer_threads" => 3})
   end
   let(:snappy_config) do
-    { 'topics' => ['logstash_integration_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1,
+    { 'topics' => ['logstash_integration_topic_snappy'], 'group_id' => group_id_1,
       'auto_offset_reset' => 'earliest' }
   end
   let(:lz4_config) do
-    { 'topics' => ['logstash_integration_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1,
+    { 'topics' => ['logstash_integration_topic_lz4'], 'group_id' => group_id_1,
       'auto_offset_reset' => 'earliest' }
   end
   let(:pattern_config) do
-    { 'topics_pattern' => 'logstash_integration_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain',
+    { 'topics_pattern' => 'logstash_integration_topic_.*', 'group_id' => group_id_2,
       'auto_offset_reset' => 'earliest' }
   end
   let(:decorate_config) do
-    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3,
+    { 'topics' => ['logstash_integration_topic_plain'], 'group_id' => group_id_3,
       'auto_offset_reset' => 'earliest', 'decorate_events' => 'true' }
   end
   let(:decorate_headers_config) do
-    { 'topics' => ['logstash_integration_topic_plain_with_headers'], 'codec' => 'plain', 'group_id' => group_id_3,
+    { 'topics' => ['logstash_integration_topic_plain_with_headers'], 'group_id' => group_id_3,
       'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
   end
   let(:decorate_bad_headers_config) do
-    { 'topics' => ['logstash_integration_topic_plain_with_headers_badly'], 'codec' => 'plain', 'group_id' => group_id_3,
+    { 'topics' => ['logstash_integration_topic_plain_with_headers_badly'], 'group_id' => group_id_3,
       'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
   end
   let(:manual_commit_config) do
-    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5,
+    { 'topics' => ['logstash_integration_topic_plain'], 'group_id' => group_id_5,
       'auto_offset_reset' => 'earliest', 'enable_auto_commit' => 'false' }
   end
   let(:timeout_seconds) { 30 }
@@ -352,10 +352,7 @@ describe "Deserializing with the schema registry", :integration => true do
 
     let(:base_config) do
       {
-          'topics' => [avro_topic_name],
-          'codec' => 'plain',
-          'group_id' => group_id_1,
-          'auto_offset_reset' => 'earliest'
+          'topics' => [avro_topic_name], 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'
       }
     end
 

data/spec/unit/inputs/kafka_spec.rb

@@ -177,7 +177,23 @@ describe LogStash::Inputs::Kafka do
     end
   end
 
-  describe "schema registry parameter verification" do
+  it 'uses plain codec by default' do
+    expect( subject.codec ).to respond_to :decode
+    expect( subject.codec.class ).to be LogStash::Codecs::Plain
+  end
+
+  context 'with codec option' do
+
+    let(:config) { super().merge 'codec' => 'line' }
+
+    it 'uses specified codec' do
+      expect( subject.codec ).to respond_to :decode
+      expect( subject.codec.class ).to be LogStash::Codecs::Line
+    end
+
+  end
+
+  describe "schema registry" do
     let(:base_config) do {
           'schema_registry_url' => 'http://localhost:8081',
           'topics' => ['logstash'],
@@ -186,7 +202,7 @@ describe LogStash::Inputs::Kafka do
     end
 
     context "schema_registry_url" do
-     let(:config) { base_config }
+      let(:config) { base_config }
 
       it "conflict with value_deserializer_class should fail" do
         config['value_deserializer_class'] = 'my.fantasy.Deserializer'
@@ -197,6 +213,11 @@ describe LogStash::Inputs::Kafka do
         config['topics_pattern'] = 'topic_.*'
         expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of topics_pattern/
       end
+
+      it 'switches default codec to json' do
+        expect( subject.codec ).to respond_to :decode
+        expect( subject.codec.class ).to be LogStash::Codecs::JSON
+      end
     end
 
     context 'when kerberos auth is used' do
@@ -204,9 +225,8 @@ describe LogStash::Inputs::Kafka do
         context "with #{protocol}" do
           ['auto', 'skip'].each do |vsr|
             context "when validata_schema_registry is #{vsr}" do
-              let(:config) { base_config.merge({'security_protocol' => protocol,
-                                                'schema_registry_validation' => vsr})
-              }
+              let(:config) { base_config.merge({'security_protocol' => protocol, 'schema_registry_validation' => vsr}) }
+
               it 'skips verification' do
                 expect(subject).not_to receive(:check_for_schema_registry_connectivity_and_subjects)
                 expect { subject.register }.not_to raise_error

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.8.2
+  version: 10.9.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-04 00:00:00.000000000 Z
+date: 2021-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -154,6 +154,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-line
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: