logstash-input-kafka 6.3.4 → 7.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 76720171728fa67ea7f524fbf533a23ac91cae85
-  data.tar.gz: 715a70f704452226da80e304f1ac56f4a5b92844
+  metadata.gz: f602cb85f24f6c18a37f9c80dbad01b5e5a17c1d
+  data.tar.gz: 5538817ea5cbd3ddc44c092e57fc35733780d7ac
 SHA512:
-  metadata.gz: b2ed0780870168cc55bf709925d39ce22fcefa4e1e509085ba1c230da1510e624f50f351fade71c0628fc7cf2e2b8363e327f87fccf39b3a9167c52048f89733
-  data.tar.gz: e9a9112d7a2d08a8b268041d690fad6d521a2267f8a1c52f06c729b03b188a0df0195bae7c38e2004e9681dcaad004b4c796b0608173d28df05edba3551a31b4
+  metadata.gz: a55bb633f746ebe5643bc1c765786f6b87807e26635ad088da30786369631833eb7b1be0b64687d6682b4c0d58f5bc3589d28718e9c299bf79b5cf115b3b0846
+  data.tar.gz: 77ec54a43e2ee2b18aa0612b206b73827fa19639c6e031f4479778c9a71374e2a8913c53a6837fd099c0658e91e4de8894393c2f7d2a5b4ab0d3d9ab0a404cc5

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 7.0.0
+  - Breaking: Nest the decorated fields under `@metadata` field to avoid mapping conflicts with beats.
+    Fixes #198, #180
+
 ## 6.3.4
   - Fix an issue that led to random failures in decoding messages when using more than one input thread
 

data/docs/index.asciidoc

@@ -64,6 +64,19 @@ For more information see http://kafka.apache.org/documentation.html#theconsumer
 
 Kafka consumer configuration: http://kafka.apache.org/documentation.html#consumerconfigs
 
+==== Metadata fields
+
+The following metadata from Kafka broker are added under the `[@metadata]` field:
+
+* `[@metadata][kafka][topic]`: Original Kafka topic from where the message was consumed.
+* `[@metadata][kafka][consumer_group]`: Consumer group
+* `[@metadata][kafka][partition]`: Partition info for this message.
+* `[@metadata][kafka][offset]`: Original record offset for this message.
+* `[@metadata][kafka][key]`: Record key, if any.
+* `[@metadata][kafka][timestamp]`: Timestamp when this message was received by the Kafka broker.
+
+Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be 
+inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`.
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Kafka Input Configuration Options

data/lib/logstash/inputs/kafka.rb

@@ -254,12 +254,12 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
             codec_instance.decode(record.value.to_s) do |event|
               decorate(event)
               if @decorate_events
-                event.set("[kafka][topic]", record.topic)
-                event.set("[kafka][consumer_group]", @group_id)
-                event.set("[kafka][partition]", record.partition)
-                event.set("[kafka][offset]", record.offset)
-                event.set("[kafka][key]", record.key)
-                event.set("[kafka][timestamp]", record.timestamp)
+                event.set("[@metadata][kafka][topic]", record.topic)
+                event.set("[@metadata][kafka][consumer_group]", @group_id)
+                event.set("[@metadata][kafka][partition]", record.partition)
+                event.set("[@metadata][kafka][offset]", record.offset)
+                event.set("[@metadata][kafka][key]", record.key)
+                event.set("[@metadata][kafka][timestamp]", record.timestamp)
               end
               logstash_queue << event
             end

data/logstash-input-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-kafka'
-  s.version         = '6.3.4'
+  s.version         = '7.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/inputs/kafka_spec.rb

@@ -134,9 +134,9 @@ describe "inputs/kafka", :integration => true do
         wait(timeout_seconds).for {queue.length}.to eq(num_events)
         expect(queue.length).to eq(num_events)
         event = queue.shift
-        expect(event.get("kafka")["topic"]).to eq("logstash_topic_plain")
-        expect(event.get("kafka")["consumer_group"]).to eq(group_id_3)
-        expect(event.get("kafka")["timestamp"]).to be >= start
+        expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_topic_plain")
+        expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
+        expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
       ensure
         t.kill
         t.join(30_000)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 6.3.4
+  version: 7.0.0
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-13 00:00:00.000000000 Z
+date: 2017-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement