logstash-codec-avro_schema_registry 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 96fba808f06019f95c4e7795b896f3d1cb583f02
-  data.tar.gz: d4dcf8c54b94c97bb591c26b6d07cc895063c5ef
+  metadata.gz: df24e437ee21154def83e8ad66bcbc70e58e3ecc
+  data.tar.gz: c7e44473770cecf6165bb38788872c45f33aa4e6
 SHA512:
-  metadata.gz: 942512a18cd5811e85621d71b7adf4a3734b1334e26ae4f8501c8a96028c57712674498c81de6ee5fcbae201c6d69a868526a643ddcf911d858e94be7f10ba3a
-  data.tar.gz: d1f85086d6d0ff6e23c8306164add731bae9ae4a2cfded29c5698a11dab6c24899c3fc19ee0613e648f9ce95c3bb64958261c4b1e292e8693a1de7683975aa13
+  metadata.gz: 56267b65a7295a4766ff67658ca2aa799609c5cfc7675bd64ec9d98ddb8848ac1eb7da8604bff67aa93cea1cc36fb435429c2addcbc77d9f3bf38e9ded7153e6
+  data.tar.gz: 210dfaf4d6eb1315d91727fb45fe1599d8f9537f0d018e0d2538a9bbae3938dab6e29da8a234dee04cbcdf0a95d137e4552ad8f4d037fbcc8cd4973d7ba0aab6

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.1.1
+ - Add tag_on_failure for decode
+
 ## 1.1.0
  - Add support for client side TLS auth to schema registry
 

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Codec - Avro Schema Registry
 
-### v1.1.0
+### v1.1.1
 
 This plugin is used to serialize Logstash events as
 Avro datums, as well as deserializing Avro datums into
@@ -22,6 +22,7 @@ When this codec is used to decode the input, you may pass the following options:
 - ``endpoint`` - always required.
 - ``username`` - optional.
 - ``password`` - optional.
+- ``tag_on_failure`` - tag events with ``_avroparsefailure`` when decode fails
 
 If the input stream is binary encoded, you should use the ``ByteArrayDeserializer``
 in the Kafka input config.

data/lib/logstash/codecs/avro_schema_registry.rb

@@ -29,6 +29,7 @@ MAGIC_BYTE = 0
 # - ``endpoint`` - always required.
 # - ``username`` - optional.
 # - ``password`` - optional.
+# - ``tag_on_failure`` - tag events with ``_avroparsefailure`` when decode fails
 #
 # If the input stream is binary encoded, you should use the ``ByteArrayDeserializer``
 # in the Kafka input config.
@@ -127,6 +128,9 @@ class LogStash::Codecs::AvroSchemaRegistry < LogStash::Codecs::Base
   config :register_schema, :validate => :boolean, :default => false
   config :binary_encoded, :validate => :boolean, :default => true
 
+  # tag events with `_avroparsefailure` when decode fails
+  config :tag_on_failure, :validate => :boolean, :default => false
+
   config :client_certificate, :validate => :string, :default => nil
   config :client_key, :validate => :string, :default => nil
   config :ca_certificate, :validate => :string, :default => nil
@@ -134,7 +138,6 @@ class LogStash::Codecs::AvroSchemaRegistry < LogStash::Codecs::Base
   
   public
   def register
-
     @client = if client_certificate != nil
       SchemaRegistry::Client.new(endpoint, username, password, SchemaRegistry::Client.connection_options(
         client_certificate: client_certificate,
@@ -200,12 +203,18 @@ class LogStash::Codecs::AvroSchemaRegistry < LogStash::Codecs::Base
 
           schema = subject.verify_schema(schema_json)
         end
-
+        # Return schema id
         schema.id
       end
     end
   end
 
+  def clean_event(event)
+    event_hash = event.to_hash
+    event_hash.delete_if { |key, _| EXCLUDE_ALWAYS.include? key }
+    event_hash
+  end
+
   public
   def decode(data)
     if data.length < 5
@@ -222,6 +231,13 @@ class LogStash::Codecs::AvroSchemaRegistry < LogStash::Codecs::Base
         yield LogStash::Event.new(datum_reader.read(decoder))
       end
     end
+  rescue => e
+    if tag_on_failure
+      @logger.error("Avro parse error, original data now in message field", :error => e)
+      yield LogStash::Event.new("message" => data, "tags" => ["_avroparsefailure"])
+    else
+      raise e
+    end
   end
 
   public
@@ -233,9 +249,7 @@ class LogStash::Codecs::AvroSchemaRegistry < LogStash::Codecs::Base
     buffer.write(MAGIC_BYTE.chr)
     buffer.write([@write_schema_id].pack("I>"))
     encoder = Avro::IO::BinaryEncoder.new(buffer)
-    eh = event.to_hash
-    eh.delete_if { |key, _| EXCLUDE_ALWAYS.include? key }
-    dw.write(eh, encoder)
+    dw.write(clean_event(event), encoder)
     if @binary_encoded
        @on_event.call(event, buffer.string.to_java_bytes)
     else

data/logstash-codec-avro_schema_registry.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-codec-avro_schema_registry'
-  s.version       = '1.1.0'
+  s.version       = '1.1.1'
   s.licenses      = ['Apache License (2.0)']
   s.summary         = "Encode and decode avro formatted data from a Confluent schema registry"
   s.description     = "Encode and decode avro formatted data from a Confluent schema registry"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-avro_schema_registry
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - RevPoint Media
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-14 00:00:00.000000000 Z
+date: 2018-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement