fluent-plugin-amqp 0.10.1 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c1f9f27a3bb14860d33be4080222b03cb00c4b4
-  data.tar.gz: 836420a8f2eb57dd430e53c3f2a581637b47f281
+  metadata.gz: 768712aa2801509d6676bd5afbed81f5871296b4
+  data.tar.gz: f0020715d5163ed87e160f2cd13f54e2ff2e549f
 SHA512:
-  metadata.gz: 291e8362d47db2d0fcb5c132f7ae033ca47b30c49b013184450de9fbaec66ab88611563bd53a5c64b6576a68e48cdf17553b78234deecd56df72a58d1dbb6761
-  data.tar.gz: 0c2aa21d0c835922ca9c5ad550761db1c95a8e4f4e5c7ada04000dec2652983570428b72559888df0f86cf81f97b5bad5e566fc1504e2b2d8ccef20be4fb95c9
+  metadata.gz: 4686083f14b7481987362864ddc535733b7c827d3839d91ecf08726afd24ff571bee3a9e8259db28f068cfa17698a4e9e85b9cdb7f4ef55442df60c83b327eca
+  data.tar.gz: 0ed03675cdb45f9c6d4723cb3135decf2577d3d06e8bfe30a9ba9ad0a2f6f01c6e8bbe094e6917c6356a5a4986cd78d6075b677b9f38fdf6cd6a5a5bd76726f1

data/lib/fluent/plugin/out_amqp.rb

@@ -44,15 +44,66 @@ module Fluent::Plugin
     config_param :tls_key, :string, default: nil
     config_param :tls_ca_certificates, :array, default: nil
     config_param :tls_verify_peer, :bool, default: true
+    config_param :content_type, :string, default: "application/octet"
+    config_param :content_encoding, :string, default: nil
+
+    config_section :header do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+    end
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
     end
 
+
+    class HeaderElement
+      include Fluent::Configurable
+
+      config_param :name, :string
+      config_param :default, :string, default: nil
+      config_param :source, default: nil  do |val|
+             if val.start_with?('[')
+              JSON.load(val)
+             else
+               val.split('.')
+            end
+         end
+
+      # Extract a header and value from the input data
+      # returning nil if value cannot be derived
+      def getValue(data)
+        val  = getNestedValue(data, @source ) if @source
+        val ||= @default if @default
+        val
+      end
+
+      def getNestedValue(data, path)
+        temp_data = data
+        temp_path = path.dup
+        until temp_data.nil? or temp_path.empty?
+          temp_data = temp_data[temp_path.shift]
+        end
+        temp_data
+      end
+    end
+
     def configure(conf)
       compat_parameters_convert(conf, :buffer)
       super
       @conf = conf
+
+      # Extract the header configuration into a collection
+      @headers = conf.elements.select {|e|
+        e.name == 'header'
+      }.map {|e|
+        he = HeaderElement.new
+        he.configure(e)
+        unless he.source || he.default
+            raise Fluent::ConfigError, "At least 'default' or 'source' must must be defined in a header configuration section."
+        end
+        he
+      }
+
       unless @host || @hosts
         raise Fluent::ConfigError, "'host' or 'hosts' must be specified."
       end
@@ -98,13 +149,27 @@ module Fluent::Plugin
       begin
         chunk.msgpack_each do |(tag, time, data)|
           begin
-            data = JSON.dump( data ) unless data.is_a?( String )
-            log.debug "Sending message #{data}, :key => #{routing_key( tag)} :headers => #{headers(tag,time)}"
-            @exch.publish(data, key: routing_key( tag ), persistent: @persistent, headers: headers( tag, time ))
-          rescue JSON::GeneratorError => e
-            log.error "Failure converting data object to json string: #{e.message}"
-            # Debug only - otherwise we may pollute the fluent logs with unparseable events and loop
-            log.debug "JSON.dump failure converting [#{data}]"
+            msg_headers = headers(tag,time,data)
+
+            begin
+              data = JSON.dump( data ) unless data.is_a?( String )
+            rescue JSON::GeneratorError => e
+              log.warn "Failure converting data object to json string: #{e.message} - sending as raw object"
+              # Debug only - otherwise we may pollute the fluent logs with unparseable events and loop
+              log.debug "JSON.dump failure converting [#{data}]"
+            end
+
+            log.debug "Sending message #{data}, :key => #{routing_key( tag)} :headers => #{headers(tag,time,data)}"
+            @exch.publish(
+              data,
+              key: routing_key( tag ),
+              persistent: @persistent,
+              headers: msg_headers,
+              content_type: @content_type,
+              content_encoding: @content_encoding)
+
+  # :nocov:
+  #  Hard to throw StandardError through test code
           rescue StandardError => e
             # This protects against invalid byteranges and other errors at a per-message level
             log.error "Unexpected error during message publishing: #{e.message}"
@@ -119,6 +184,7 @@ module Fluent::Plugin
         # Just in case theres any other errors during chunk loading.
         log.error "Unexpected error during message publishing: #{e.message}"
       end
+      # :nocov:
     end
 
 
@@ -130,15 +196,25 @@ module Fluent::Plugin
       end
     end
 
-    def headers( tag, time )
-      {}.tap do |h|
-        h[@tag_header] = tag if @tag_header
-        h[@time_header] = Time.at(time).utc.to_s if @time_header
-      end
+    def headers( tag, time, data )
+      h = {}
+
+      log.debug "Processing Headers: #{@headers}"
+      # A little messy this...
+      # Trying to allow for header overrides where a header defined
+      # earlier will be used if a later header is returning nil (ie not found and no default)
+      h = Hash[ @headers
+                  .collect{|v| [v.name, v.getValue(data) ]}
+                  .delete_if{|x| x.last.nil?}
+          ]
+
+      h[@tag_header] = tag if @tag_header
+      h[@time_header] = Time.at(time).utc.to_s if @time_header
+
+      h
     end
 
 
-    private
     def check_tls_configuration()
       if @tls
         unless @tls_key && @tls_cert
@@ -161,6 +237,5 @@ module Fluent::Plugin
       opts[:tls_ca_certificates] = @tls_ca_certificates if @tls_ca_certificates
       return opts
     end
-
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-amqp
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.11.0
 platform: ruby
 authors:
 - Hiromi Ishii
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-17 00:00:00.000000000 Z
+date: 2017-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -167,7 +167,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: AMQP input/output plugin or fluentd