fluent-plugin-avro_deluxe 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a0e4f8b9b1274e6614a9b34371662de4b32f6aae
+  data.tar.gz: 0d19cb5cfd89d1f1a102bb8e238bd7bc42d08a4f
+SHA512:
+  metadata.gz: 4f648bf32dec3de55670c3cdc50995adedb630618a12c25dc2ef541db3a51f9d66462a204a0d5fe4d5933d10fb37f8f22c0ded5c85ddfcba5f023afeb5d3627a
+  data.tar.gz: 0c6f5d903bc9a862b99e00835af656c8bb714485b537ca2c471dfb585d85a6f06d6d7917fc366b68c391a223060257e8486f48c9dba5d1123074a50d9154bc40

data/lib/formatter_avro.rb

@@ -0,0 +1,79 @@
+require 'avro'
+require 'schema_registry'
+# https://github.com/revpoint/logstash-codec-avro_schema_registry/blob/master/lib/logstash/codecs/avro_schema_registry.rb
+module Fluent
+  module TextFormatter
+    class AvroFormatter < Formatter
+      Fluent::Plugin.register_formatter('avro', self)
+
+      config_param :schema_file, :string, :default => nil
+      config_param :schema_json, :string, :default => nil
+      config_param :schema_url, :string, :default => nil
+      config_param :schema_subject, :string, :default => nil
+      config_param :schema_version, :string, :default => nil
+      # this stuff is needed if you're using a schema registry
+      # config_param :schema_version, :string, :default => nil
+      config_param :schema_id, :string, :default => nil
+      def configure(conf)
+        super
+        if !@schema_file.nil? then
+          raise Fluent::ConfigError "schema_file specified but schema_json already set" if !@schema_json.nil?
+          @schema_json = File.read(@schema_file)
+        end
+        if !@schema_url.nil? then
+          raise Fluent::ConfigError "schema_url specified but schema_json already set" if !@schema_json.nil?
+          client = SchemaRegistry::Client.new(
+            @schema_url,
+            '',
+            '', 
+            SchemaRegistry::Client.connection_options(
+              client_certificate: '/etc/td-agent/ssl/avro_client.crt', 
+              client_key: '/etc/td-agent/ssl/avro_client.key',
+              ca_certificate: "/etc/td-agent/ssl/avro_ca.pem",
+              verify_mode: :verify_peer
+            )
+          )
+          subject = client.subject(@schema_subject)
+          schema = subject.version(@schema_version)
+          @schema_json = schema.schema
+          @schema_id = schema.id
+          # puts "id:#{@schema_id} #{@schema_json}"
+          # body = Net::HTTP.get(URI.parse(@schema_url))
+          # data = JSON.parse(body)
+          # @schema_version = data['version']
+          # @schema_id = data['id']
+          # @schema_json = JSON.parse(data['schema']).to_json
+        end
+        puts "formatter_avro initializing with schema id #{@schema_id} json: #{@schema_json}"
+        @schema = Avro::Schema.parse(@schema_json)
+      end
+
+      def format(tag, time, record)
+        # <hack offender="rolabrie@cisco.com" technical_debt>
+        # We needed tags to be an array to satisfy the avro schema
+        # fluentd 0.12 doesn't support array type for record object attribs
+        # so instead of writing a whole new formatter plugin to handle this
+        # we're just clobbering it here -- therefore tags must be set
+        # which we are doing inside 990_transform.conf
+        
+        record['tags'] = (record['tags'] || '').split(',')
+        # </hack>
+
+        # uncomment this when troubleshooting
+        # puts "formatter_avro handling type:#{record['type']} kafka_topic:#{record['kafka_topic']} payload:\n#{record['payload']}"
+        magic_byte = 0
+        dw = Avro::IO::DatumWriter.new(@schema)
+        buffer = StringIO.new
+        if @schema_url
+          # this only matters if using a schema registry
+          buffer.write(magic_byte.chr)
+          buffer.write([@schema_id].pack("I>"))
+        end
+        encoder = Avro::IO::BinaryEncoder.new(buffer)
+        eh = record.to_hash
+        dw.write(eh, encoder)
+        buffer.string
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,43 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-avro_deluxe
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Robert Labrie
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-10-25 00:00:00.000000000 Z
+dependencies: []
+description: Fluentd avro formnatter - Do not use this unsupported module
+email: rolabrie@cisco.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/formatter_avro.rb
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 3
+summary: Do not use this unsupported module
+test_files: []