fluent-plugin-amqp-sboagibm 0.21.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 90703c76c9a22638888c0893e2d4224d862b64be
+  data.tar.gz: 1875e505fcc59cc79c21f648871f7b38aed47f87
+SHA512:
+  metadata.gz: 0d99585494a1a67549c8e1a61ac6a2c76c483f6991aaf4c569b3849dbbae984e24abb5c78bd212fc41cfcb084b377625679ae11b446b8eccfba03e5de29c31dd
+  data.tar.gz: 535d4d98895e3eae957a7b00aa1dfe11e7c7019db44143eb5642a3bf2baaabf23407b855cc66c776109a85f4d198dcd8971733b4474a3ac9c802b84b3c618e89

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Hiromi Ishii, 2013- github/giraffi
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/fluent/plugin/in_amqp.rb

@@ -0,0 +1,164 @@
+require 'time'
+require 'fluent/plugin/input'
+require 'fluent/plugin/parser'
+require 'bunny'
+
+module Fluent::Plugin
+  ##
+  # AMQPInput to be used as a Fluent SOURCE, reading messages from a RabbitMQ
+  # message broker
+  class AMQPInput < Input
+    Fluent::Plugin.register_input('amqp', self)
+
+    helpers :compat_parameters, :parser
+
+    # Bunny connection handle
+    #   - Allows mocking for test purposes
+    attr_accessor :connection
+
+    config_param :tag, :string, default: "hunter.amqp"
+
+    config_param :host, :string, default: nil
+    config_param :hosts, :array, default: nil
+    config_param :user, :string, default: "guest"
+    config_param :pass, :string, default: "guest", secret: true
+    config_param :vhost, :string, default: "/"
+    config_param :port, :integer, default: 5672
+    config_param :ssl, :bool, default: false
+    config_param :verify_ssl, :bool, default: false
+    config_param :heartbeat, :integer, default: 60
+    config_param :queue, :string, default: nil
+    config_param :durable, :bool, default: false
+    config_param :exclusive, :bool, default: false
+    config_param :auto_delete, :bool, default: false
+    config_param :passive, :bool, default: false
+    config_param :payload_format, :string, default: "json"
+    config_param :tag_key, :bool, default: false
+    config_param :tag_header, :string, default: nil
+    config_param :time_header, :string, default: nil
+    config_param :tls, :bool, default: false
+    config_param :tls_cert, :string, default: nil
+    config_param :tls_key, :string, default: nil
+    config_param :tls_ca_certificates, :array, default: nil
+    config_param :tls_verify_peer, :bool, default: true
+    config_param :bind_exchange, :bool, default: false
+    config_param :exchange, :string, default: ""
+    config_param :routing_key, :string, default: "#"                       # The routing key used to bind queue to exchange - # = matches all, * matches section (tag.*.info)
+
+    def configure(conf)
+      conf['format'] ||= conf['payload_format'] # legacy
+      compat_parameters_convert(conf, :parser)
+
+      super
+
+      parser_config = conf.elements('parse').first
+      if parser_config
+        @parser = parser_create(conf: parser_config)
+      end
+
+      @conf = conf
+      unless (@host || @hosts) && @queue
+        raise Fluent::ConfigError, "'host(s)' and 'queue' must be all specified."
+      end
+      check_tls_configuration
+    end
+
+    def start
+      super
+      # Create a new connection, unless its already been provided to us
+      @connection = Bunny.new get_connection_options unless @connection
+      @connection.start
+      @channel = @connection.create_channel
+
+      if @exclusive && fluentd_worker_id > 0
+        log.info 'Config requested exclusive queue with multiple workers'
+        @queue += ".#{fluentd_worker_id}"
+        log.info "Renamed queue name to include worker id: #{@queue}"
+      end
+
+      q = @channel.queue(@queue, passive: @passive, durable: @durable,
+                       exclusive: @exclusive, auto_delete: @auto_delete)
+      if @bind_exchange
+        log.info "Binding #{@queue} to #{@exchange}, :routing_key => #{@routing_key}"
+        q.bind(exchange=@exchange, routing_key: @routing_key)
+      end
+
+      q.subscribe do |delivery, meta, msg|
+        log.debug "Recieved message #{@msg}"
+        payload = parse_payload(msg)
+        router.emit(parse_tag(delivery, meta), parse_time(meta), payload)
+      end
+    end # AMQPInput#run
+
+    def shutdown
+      log.info "Closing connection"
+      @connection.stop
+      super
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    private
+    def parse_payload(msg)
+      if @parser
+        parsed = nil
+        @parser.parse msg do |_, payload|
+          if payload.nil?
+            log.warn "failed to parse #{msg}"
+            parsed = { "message" => msg }
+          else
+            parsed = payload
+          end
+        end
+        parsed
+      else
+        { "message" => msg }
+      end
+    end
+
+    def parse_tag( delivery, meta )
+      if @tag_key && delivery.routing_key != ''
+        delivery.routing_key
+      elsif @tag_header && meta[:headers][@tag_header]
+        meta[:headers][@tag_header]
+      else
+        @tag
+      end
+    end
+
+    def parse_time( meta )
+      if @time_header && meta[:headers][@time_header]
+        Fluent::EventTime.from_time(Time.parse( meta[:headers][@time_header] ))
+      else
+        Fluent::Engine.now
+      end
+    end
+
+    def check_tls_configuration()
+      if @tls
+        unless @tls_key && @tls_cert
+            raise Fluent::ConfigError, "'tls_key' and 'tls_cert' must be all specified if tls is enabled."
+        end
+      end
+    end
+
+    def get_connection_options()
+      hosts = @hosts ||= Array.new(1, @host)
+      opts = {
+        hosts: hosts, port: @port, vhost: @vhost,
+        pass: @pass, user: @user, ssl: @ssl,
+        verify_ssl: @verify_ssl, heartbeat: @heartbeat,
+        tls: @tls,
+        tls_cert: @tls_cert,
+        tls_key: @tls_key,
+        verify_peer: @tls_verify_peer
+      }
+      opts[:tls_ca_certificates] = @tls_ca_certificates if @tls_ca_certificates
+      return opts
+    end
+
+  end # class AMQPInput
+
+end # module Fluent::Plugin

data/lib/fluent/plugin/out_amqp.rb

@@ -0,0 +1,262 @@
+require 'json'
+require 'fluent/plugin/output'
+require "bunny"
+
+module Fluent::Plugin
+  ##
+  # AMQPOutput to be used as a Fluent MATCHER, sending messages to a RabbitMQ
+  # messaging broker
+  class AMQPOutput < Output
+    Fluent::Plugin.register_output("amqp", self)
+
+    helpers :compat_parameters
+
+    DEFAULT_BUFFER_TYPE = "memory"
+
+    attr_accessor :connection
+
+    #Attribute readers to support testing
+    attr_reader :exch
+    attr_reader :channel
+
+
+    config_param :host, :string, default: nil
+    config_param :hosts, :array, default: nil
+    config_param :user, :string, default: "guest"
+    config_param :pass, :string, default: "guest", secret: true
+    config_param :vhost, :string, default: "/"
+    config_param :port, :integer, default: 5672
+    config_param :ssl, :bool, default: false
+    config_param :verify_ssl, :bool, default: false
+    config_param :heartbeat, :integer, default: 60
+    config_param :exchange, :string, default: ""
+    config_param :exchange_type, :string, default: "direct"
+    config_param :passive, :bool, default: false
+    config_param :durable, :bool, default: false
+    config_param :auto_delete, :bool, default: false
+    config_param :key, :string, default: nil
+    config_param :persistent, :bool, default: false
+    config_param :tag_key, :bool, default: false
+    config_param :tag_header, :string, default: nil
+    config_param :time_header, :string, default: nil
+    config_param :tls, :bool, default: false
+    config_param :tls_cert, :string, default: nil
+    config_param :tls_key, :string, default: nil
+    config_param :tls_ca_certificates, :array, default: nil
+    config_param :tls_verify_peer, :bool, default: true
+    config_param :content_type, :string, default: "application/octet"
+    config_param :content_encoding, :string, default: nil
+
+    config_section :header do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+    end
+
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+    end
+
+
+    class HeaderElement
+      include Fluent::Configurable
+
+      config_param :name, :string
+      config_param :default, :string, default: nil
+      config_param :source, default: nil  do |val|
+           if val.start_with?('[')
+             JSON.load(val)
+           else
+             val.split('.')
+          end
+         end
+
+      # Extract a header and value from the input data
+      # returning nil if value cannot be derived
+      def getValue(data)
+        val  = getNestedValue(data, @source ) if @source
+        val ||= @default if @default
+        val
+      end
+
+      def getNestedValue(data, path)
+        temp_data = data
+        temp_path = path.dup
+        until temp_data.nil? or temp_path.empty?
+          temp_data = temp_data[temp_path.shift]
+        end
+        temp_data
+      end
+    end
+
+    def configure(conf)
+      compat_parameters_convert(conf, :buffer)
+      super
+      @conf = conf
+
+      # Extract the header configuration into a collection
+      @headers = conf.elements.select {|e|
+        e.name == 'header'
+      }.map {|e|
+        he = HeaderElement.new
+        he.configure(e)
+        unless he.source || he.default
+          raise Fluent::ConfigError, "At least 'default' or 'source' must must be defined in a header configuration section."
+        end
+        he
+      }
+
+      unless @host || @hosts
+        raise Fluent::ConfigError, "'host' or 'hosts' must be specified."
+      end
+      unless @key || @tag_key
+        raise Fluent::ConfigError, "Either 'key' or 'tag_key' must be set."
+      end
+      check_tls_configuration
+    end
+
+    def start
+      super
+      begin
+        log.info "Connecting to RabbitMQ..."
+        @connection = Bunny.new(get_connection_options) unless @connection
+        @connection.start
+      rescue Bunny::TCPConnectionFailed => e
+        log.error "Connection to #{@host} failed"
+      rescue Bunny::PossibleAuthenticationFailureError => e
+        log.error "Could not authenticate as #{@user}"
+      end
+
+      log.info 'Creating new channel'
+      @channel = @connection.create_channel
+
+      return if @exchange.to_s =~ CHUNK_KEY_PLACEHOLDER_PATTERN
+
+      log.info 'Creating new exchange (in start)', exchange: @exchange
+      @exch = @channel.exchange(@exchange, type: @exchange_type.intern,
+                              passive: @passive, durable: @durable,
+                              auto_delete: @auto_delete)
+    end
+
+    def shutdown
+      @connection.stop
+      super
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def formatted_to_msgpack_binary
+      true
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      begin
+        log.debug 'in write', chunk_id: dump_unique_id_hex(chunk.unique_id)
+        log.debug 'raw exchange value is', exchange: @exchange.to_s
+
+        if @exchange.to_s =~ CHUNK_KEY_PLACEHOLDER_PATTERN
+          exchange_name = extract_placeholders(@exchange, chunk)
+          log.info 'resolved exchange value is', exchange_name: exchange_name
+          @exch = @channel.exchange(exchange_name, type: @exchange_type.intern,
+                                    passive: @passive, durable: @durable,
+                                    auto_delete: @auto_delete)
+        end
+
+        log.debug 'writing data to exchange', chunk_id: dump_unique_id_hex(chunk.unique_id)
+
+        chunk.msgpack_each do |(tag, time, data)|
+          begin
+            msg_headers = headers(tag,time,data)
+
+            begin
+              data = JSON.dump( data ) unless data.is_a?( String )
+            rescue JSON::GeneratorError => e
+              log.warn "Failure converting data object to json string: #{e.message} - sending as raw object"
+              # Debug only - otherwise we may pollute the fluent logs with unparseable events and loop
+              log.debug "JSON.dump failure converting [#{data}]"
+            end
+
+            log.info "Sending message #{data}, :key => #{routing_key( tag)} :headers => #{headers(tag,time,data)}"
+            @exch.publish(
+              data,
+              key: routing_key( tag ),
+              persistent: @persistent,
+              headers: msg_headers,
+              content_type: @content_type,
+              content_encoding: @content_encoding)
+
+  # :nocov:
+  #  Hard to throw StandardError through test code
+          rescue StandardError => e
+            # This protects against invalid byteranges and other errors at a per-message level
+            log.error "Unexpected error during message publishing: #{e.message}"
+            log.debug "Failure in publishing message [#{data}]"
+          end
+        end
+      rescue MessagePack::MalformedFormatError => e
+        # This has been observed when a server has filled the partition containing
+        # the buffer files, and during replay the chunks were malformed
+        log.error "Malformed msgpack in chunk - Did your server run out of space during buffering? #{e.message}"
+      rescue StandardError => e
+        # Just in case theres any other errors during chunk loading.
+        log.error "Unexpected error during message publishing: #{e.message}"
+      end
+      # :nocov:
+    end
+
+
+    def routing_key( tag )
+      if @tag_key
+        tag
+      else
+        @key
+      end
+    end
+
+    def headers( tag, time, data )
+      h = {}
+
+      log.debug "Processing Headers: #{@headers}"
+      # A little messy this...
+      # Trying to allow for header overrides where a header defined
+      # earlier will be used if a later header is returning nil (ie not found and no default)
+      h = Hash[ @headers
+                  .collect{|v| [v.name, v.getValue(data) ]}
+                  .delete_if{|x| x.last.nil?}
+          ]
+
+      h[@tag_header] = tag if @tag_header
+      h[@time_header] = Time.at(time).utc.to_s if @time_header
+
+      h
+    end
+
+
+    def check_tls_configuration()
+      if @tls
+        unless @tls_key && @tls_cert
+          raise Fluent::ConfigError, "'tls_key' and 'tls_cert' must be all specified if tls is enabled."
+        end
+      end
+    end
+
+    def get_connection_options()
+      hosts = @hosts ||= Array.new(1, @host)
+      opts = {
+        hosts: hosts, port: @port, vhost: @vhost,
+        pass: @pass, user: @user, ssl: @ssl,
+        verify_ssl: @verify_ssl, heartbeat: @heartbeat,
+        tls: @tls || nil,
+        tls_cert: @tls_cert,
+        tls_key: @tls_key,
+        verify_peer: @tls_verify_peer
+      }
+      opts[:tls_ca_certificates] = @tls_ca_certificates if @tls_ca_certificates
+      return opts
+    end
+  end
+end

metadata

@@ -0,0 +1,174 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-amqp-sboagibm
+version: !ruby/object:Gem::Version
+  version: 0.21.0
+platform: ruby
+authors:
+- Hiromi Ishii
+- Team Giraffi
+- HiganWorks LLC
+- Toby Jackson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-11-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.8
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.8
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: bunny-mock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.5.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.10'
+description: AMQP input/output plugin for fluentd
+email: sawanoboriyu@higanworks.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+files:
+- LICENSE.txt
+- lib/fluent/plugin/in_amqp.rb
+- lib/fluent/plugin/out_amqp.rb
+homepage: https://github.com/sboagibm/fluent-plugin-amqp
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.3
+signing_key: 
+specification_version: 4
+summary: AMQP input/output plugin or fluentd
+test_files: []