fluent-plugin-amqp 0.10.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b4afb9f6144c0448aefb570fbe50d47f8ad4845c
-  data.tar.gz: 9c07383eb91e5e825d12fb23a93b294245afb9c8
+SHA256:
+  metadata.gz: 83133837639cc240c17c319ddc075ff365e4ed599e393439216f00a7ba62ec12
+  data.tar.gz: 51005871d326dd9ee209f77e96c602892e2ef8bda5a126753007bd5e82770470
 SHA512:
-  metadata.gz: dc9872fd8d8fef5e4f38bebacbf440a7b6b89568f4285592d09190d4ecdf40ec32f9f94606f6f273a7e6ca1b2737d53d0aafc237a9b5002c62f41da07e006ab7
-  data.tar.gz: 972dcc4296cc3e0c87245fde0763336a39e8e436237ec3f7930516e68ca0db9c13aa7e19714ef19549d8770192c595960737c242a0c13da38b759effcbd8d48c
+  metadata.gz: 8bd4d7ca5e802b74fcee42bd7050dc63e20d4ee0fdbbf15ab8b3084df3c302ec674b1452ada1ff19ba48a0d4c59affc191f71bd93f35277b115d0aff42fef228
+  data.tar.gz: 88494f525f1b9c6d3b211068e1a3a5ee04b4dc03a34dded4415534046e6329fe9ad480e04cc12d4ea08104c3837d8f2ae7c4f70d0fd09d64a76b1f5930a70b6d

data/lib/fluent/plugin/in_amqp.rb

@@ -44,6 +44,8 @@ module Fluent::Plugin
     config_param :bind_exchange, :bool, default: false
     config_param :exchange, :string, default: ""
     config_param :routing_key, :string, default: "#"                       # The routing key used to bind queue to exchange - # = matches all, * matches section (tag.*.info)
+    config_param :include_headers, :bool, default: false
+    config_param :auth_mechanism, :string, default: nil
 
     def configure(conf)
       conf['format'] ||= conf['payload_format'] # legacy
@@ -60,7 +62,6 @@ module Fluent::Plugin
       unless (@host || @hosts) && @queue
         raise Fluent::ConfigError, "'host(s)' and 'queue' must be all specified."
       end
-      check_tls_configuration
     end
 
     def start
@@ -69,6 +70,13 @@ module Fluent::Plugin
       @connection = Bunny.new get_connection_options unless @connection
       @connection.start
       @channel = @connection.create_channel
+
+      if @exclusive && fluentd_worker_id > 0
+        log.info 'Config requested exclusive queue with multiple workers'
+        @queue += ".#{fluentd_worker_id}"
+        log.info "Renamed queue name to include worker id: #{@queue}"
+      end
+
       q = @channel.queue(@queue, passive: @passive, durable: @durable,
                        exclusive: @exclusive, auto_delete: @auto_delete)
       if @bind_exchange
@@ -77,8 +85,10 @@ module Fluent::Plugin
       end
 
       q.subscribe do |delivery, meta, msg|
-        log.debug "Recieved message #{@msg}"
-        payload = parse_payload(msg)
+        log.debug "Recieved message #{msg}"
+        log.debug "Recieved message  MetaData #{meta}"
+        payload = parse_payload(msg, meta)
+        log.debug "Parsed Payload #{payload}"
         router.emit(parse_tag(delivery, meta), parse_time(meta), payload)
       end
     end # AMQPInput#run
@@ -89,21 +99,30 @@ module Fluent::Plugin
       super
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     private
-    def parse_payload(msg)
+    def parse_payload(msg, meta)
+      parsed = nil
       if @parser
-        parsed = nil
         @parser.parse msg do |_, payload|
           if payload.nil?
             log.warn "failed to parse #{msg}"
-            parsed = { "message" => msg }
+            parsed = { 'message' => msg }
           else
             parsed = payload
           end
         end
-        parsed
       else
-        { "message" => msg }
+        parsed = { 'message' => msg }
+      end
+      if @include_headers
+        log.debug 'Adding headers'
+        { 'headers' => meta[:headers] }.merge(parsed)
+      else
+        parsed
       end
     end
 
@@ -125,14 +144,6 @@ module Fluent::Plugin
       end
     end
 
-    def check_tls_configuration()
-      if @tls
-        unless @tls_key && @tls_cert
-            raise Fluent::ConfigError, "'tls_key' and 'tls_cert' must be all specified if tls is enabled."
-        end
-      end
-    end
-
     def get_connection_options()
       hosts = @hosts ||= Array.new(1, @host)
       opts = {
@@ -140,10 +151,12 @@ module Fluent::Plugin
         pass: @pass, user: @user, ssl: @ssl,
         verify_ssl: @verify_ssl, heartbeat: @heartbeat,
         tls: @tls,
-        tls_cert: @tls_cert,
-        tls_key: @tls_key,
-        verify_peer: @tls_verify_peer
+        verify_peer: @tls_verify_peer,
+        auth_mechanism: @auth_mechanism
       }
+      # Include additional optional TLS configurations
+      opts[:tls_key] = @tls_key if @tls_key
+      opts[:tls_cert] = @tls_cert if @tls_cert
       opts[:tls_ca_certificates] = @tls_ca_certificates if @tls_ca_certificates
       return opts
     end

data/lib/fluent/plugin/out_amqp.rb

@@ -44,22 +44,73 @@ module Fluent::Plugin
     config_param :tls_key, :string, default: nil
     config_param :tls_ca_certificates, :array, default: nil
     config_param :tls_verify_peer, :bool, default: true
+    config_param :content_type, :string, default: "application/octet"
+    config_param :content_encoding, :string, default: nil
+    config_param :auth_mechanism, :string, default: nil
+
+    config_section :header do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+    end
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
     end
 
+
+    class HeaderElement
+      include Fluent::Configurable
+
+      config_param :name, :string
+      config_param :default, :string, default: nil
+      config_param :source, default: nil  do |val|
+             if val.start_with?('[')
+              JSON.load(val)
+             else
+               val.split('.')
+            end
+         end
+
+      # Extract a header and value from the input data
+      # returning nil if value cannot be derived
+      def getValue(data)
+        val  = getNestedValue(data, @source ) if @source
+        val ||= @default if @default
+        val
+      end
+
+      def getNestedValue(data, path)
+        temp_data = data
+        temp_path = path.dup
+        until temp_data.nil? or temp_path.empty?
+          temp_data = temp_data[temp_path.shift]
+        end
+        temp_data
+      end
+    end
+
     def configure(conf)
       compat_parameters_convert(conf, :buffer)
       super
       @conf = conf
+
+      # Extract the header configuration into a collection
+      @headers = conf.elements.select {|e|
+        e.name == 'header'
+      }.map {|e|
+        he = HeaderElement.new
+        he.configure(e)
+        unless he.source || he.default
+            raise Fluent::ConfigError, "At least 'default' or 'source' must must be defined in a header configuration section."
+        end
+        he
+      }
+
       unless @host || @hosts
         raise Fluent::ConfigError, "'host' or 'hosts' must be specified."
       end
       unless @key || @tag_key
         raise Fluent::ConfigError, "Either 'key' or 'tag_key' must be set."
       end
-      check_tls_configuration
     end
 
     def start
@@ -86,6 +137,10 @@ module Fluent::Plugin
       super
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def formatted_to_msgpack_binary
       true
     end
@@ -98,13 +153,27 @@ module Fluent::Plugin
       begin
         chunk.msgpack_each do |(tag, time, data)|
           begin
-            data = JSON.dump( data ) unless data.is_a?( String )
-            log.debug "Sending message #{data}, :key => #{routing_key( tag)} :headers => #{headers(tag,time)}"
-            @exch.publish(data, key: routing_key( tag ), persistent: @persistent, headers: headers( tag, time ))
-          rescue JSON::GeneratorError => e
-            log.error "Failure converting data object to json string: #{e.message}"
-            # Debug only - otherwise we may pollute the fluent logs with unparseable events and loop
-            log.debug "JSON.dump failure converting [#{data}]"
+            msg_headers = headers(tag,time,data)
+
+            begin
+              data = JSON.dump( data ) unless data.is_a?( String )
+            rescue JSON::GeneratorError => e
+              log.warn "Failure converting data object to json string: #{e.message} - sending as raw object"
+              # Debug only - otherwise we may pollute the fluent logs with unparseable events and loop
+              log.debug "JSON.dump failure converting [#{data}]"
+            end
+
+            log.debug "Sending message #{data}, :key => #{routing_key( tag)} :headers => #{headers(tag,time,data)}"
+            @exch.publish(
+              data,
+              key: routing_key( tag ),
+              persistent: @persistent,
+              headers: msg_headers,
+              content_type: @content_type,
+              content_encoding: @content_encoding)
+
+  # :nocov:
+  #  Hard to throw StandardError through test code
           rescue StandardError => e
             # This protects against invalid byteranges and other errors at a per-message level
             log.error "Unexpected error during message publishing: #{e.message}"
@@ -119,6 +188,7 @@ module Fluent::Plugin
         # Just in case theres any other errors during chunk loading.
         log.error "Unexpected error during message publishing: #{e.message}"
       end
+      # :nocov:
     end
 
 
@@ -130,21 +200,22 @@ module Fluent::Plugin
       end
     end
 
-    def headers( tag, time )
-      {}.tap do |h|
-        h[@tag_header] = tag if @tag_header
-        h[@time_header] = Time.at(time).utc.to_s if @time_header
-      end
-    end
+    def headers( tag, time, data )
+      h = {}
 
+      log.debug "Processing Headers: #{@headers}"
+      # A little messy this...
+      # Trying to allow for header overrides where a header defined
+      # earlier will be used if a later header is returning nil (ie not found and no default)
+      h = Hash[ @headers
+                  .collect{|v| [v.name, v.getValue(data) ]}
+                  .delete_if{|x| x.last.nil?}
+          ]
 
-    private
-    def check_tls_configuration()
-      if @tls
-        unless @tls_key && @tls_cert
-            raise Fluent::ConfigError, "'tls_key' and 'tls_cert' must be all specified if tls is enabled."
-        end
-      end
+      h[@tag_header] = tag if @tag_header
+      h[@time_header] = Time.at(time).utc.to_s if @time_header
+
+      h
     end
 
     def get_connection_options()
@@ -156,11 +227,11 @@ module Fluent::Plugin
         tls: @tls || nil,
         tls_cert: @tls_cert,
         tls_key: @tls_key,
-        verify_peer: @tls_verify_peer
+        verify_peer: @tls_verify_peer,
+        auth_mechanism: @auth_mechanism
       }
       opts[:tls_ca_certificates] = @tls_ca_certificates if @tls_ca_certificates
       return opts
     end
-
   end
 end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-amqp
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Hiromi Ishii
 - Team Giraffi
 - HiganWorks LLC
 - Toby Jackson
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-17 00:00:00.000000000 Z
+date: 2020-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -151,7 +151,7 @@ homepage: http://github.com/giraffi/fluent-plugin-amqp
 licenses:
 - Apache License, Version 2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -159,16 +159,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.1.0
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: AMQP input/output plugin or fluentd
 test_files: []