logstash-codec-avro-data-file 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fedde9770f0cee9dc35a13fb9a9b4a4a7714cbd1
-  data.tar.gz: 2086eb43ef61413f865db0fa1ad9ad945969443e
+  metadata.gz: 7800340e990fa0ec4cdfade51aef94f8ca6ae361
+  data.tar.gz: d114301643eca8e6279f3c0252a3fc6fbc841ffc
 SHA512:
-  metadata.gz: d6af841762dace1ad1c4cc7f0108bdc30a64f8ae5abd63d1fef7c06ffb31eaeb3ed01b46ef95a138d6a83bfc9b6511cc151241e2aaa4552ff4c68dbe8046d8b1
-  data.tar.gz: 6b40bd57abef79f2ea29294a577e424809bacf2c2c52dca762339aa698bebd9548780a3967e6b42dd6fbf98cbfc94a43ae374dc65dcd8773e517a6fb96ad22f6
+  metadata.gz: 1baadb595507445ac476be560738cc4b75239ad7141ae41731ee48e37663b70c2cc36e203ec6f8be5adcbf93c9dc2e27003b963144f2d04993ac92d48b30be14
+  data.tar.gz: 12d9e95dffa44cc8125ae7f99a94eeed15a958719b908ca5624f1328b04500b04eb777efe2bb63337367b63af2036f4a8047172a6157c36a806a85926ed791d9

data/lib/logstash/codecs/avro-data-file.rb

@@ -5,19 +5,21 @@ require 'logstash/codecs/base'
 require 'logstash/namespace'
 require 'tmpdir'
 
-# This  codec will append a string to the message field
-# of an event, either in the decoding or encoding methods
+# == Logstash Codec - Avro Data File
+# 
+# This plugin is used to process logstash events that represent
+# Avro data files, like the S3 input can produce.
 #
-# This is only intended to be used as an example.
+# ==== Options
 #
-# input {
-#   stdin { codec => 'avro-data-file' }
-# }
+# - ``temporary_directory`` - optional. Specifies a directory to store
+#   temporary files in
+# - ``decorate_events`` - will add avro schema metadata to the events.
 #
-# or
+# ==== Usage
 #
-# output {
-#   stdout { codec => 'avro-data-file' }
+# input {
+#   stdin { codec => 'avro-data-file' }
 # }
 #
 class LogStash::Codecs::AvroDataFile < LogStash::Codecs::Base
@@ -27,6 +29,7 @@ class LogStash::Codecs::AvroDataFile < LogStash::Codecs::Base
   # Set the directory where logstash will store the tmp files before processing them.
   # default to the current OS temporary directory in linux /tmp/logstash/avro
   config :temporary_directory, validate: :string, default: File.join(Dir.tmpdir, 'logstash', 'avro')
+  config :decorate_events, validate: :boolean, default: false
 
   def register
     require 'fileutils'
@@ -43,11 +46,15 @@ class LogStash::Codecs::AvroDataFile < LogStash::Codecs::Base
     return unless block_given?
 
     Avro::DataFile.open(tempfile.path, 'r') do |reader|
+      schema = reader.datum_reader.writers_schema
       reader.each do |avro_message|
-        yield LogStash::Event.new(avro_message)
+        event = LogStash::Event.new(avro_message)
+        decorate_event(event, schema) if decorate_events?
+        yield event
       end
     end
   rescue => e
+    puts e
     @logger.error('Avro parse error', error: e)
   ensure
     reset
@@ -76,4 +83,16 @@ class LogStash::Codecs::AvroDataFile < LogStash::Codecs::Base
     end
     self.tempfile = Tempfile.create('', temporary_directory)
   end
+
+  def decorate_events?
+    @decorate_events
+  end
+
+  def decorate_event(event, schema)
+    event.set('[@metadata][avro][type]', schema.type)
+    if schema.is_a?(Avro::Schema::NamedSchema)
+      event.set('[@metadata][avro][name]', schema.name)
+      event.set('[@metadata][avro][namespace]', schema.namespace)
+    end
+  end
 end

data/logstash-codec-avro-data-file.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-codec-avro-data-file'
-  s.version       = '0.1.0'
+  s.version       = '0.2.0'
   s.licenses      = ['MIT']
   s.summary       = 'Codec for parsing avro data files'
   s.homepage      = 'https://github.com/salsify/logstash-codec-avro-data-file'

data/spec/log_stash/codecs/avro_data_file_spec.rb

@@ -52,11 +52,14 @@ describe LogStash::Codecs::AvroDataFile do
       end
 
       context "valid data" do
+        let(:schema_name) { 'mock_schema' }
+        let(:schema_type) { 'record' }
+        let(:schema_namespace) { 'com.salsify.test' }
         let(:schema) do
           {
-            'type' => 'record',
-            'name' => 'mock_schema',
-            'namespace' => 'com.salsify.test',
+            'type' => schema_type,
+            'name' => schema_name,
+            'namespace' => schema_namespace,
             'fields' => [
               {
                 'name' => 'id',
@@ -85,6 +88,21 @@ describe LogStash::Codecs::AvroDataFile do
         let(:expected_events) { avro_messages.map(&LogStash::Event.method(:new)) }
 
         include_examples "produces the correct output"
+
+        context "decorate events" do
+          let(:config) { { 'decorate_events' => true } }
+          let(:expected_events) do
+            avro_messages.map do |message|
+              LogStash::Event.new(message).tap do |event|
+                event.set('[@metadata][avro][type]', schema_type)
+                event.set('[@metadata][avro][name]', schema_name)
+                event.set('[@metadata][avro][namespace]', schema_namespace)
+              end
+            end
+          end
+
+          include_examples "produces the correct output"
+        end
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-avro-data-file
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Kyle Phelps