fluent-plugin-amqp-sboagibm 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 90703c76c9a22638888c0893e2d4224d862b64be
4
+ data.tar.gz: 1875e505fcc59cc79c21f648871f7b38aed47f87
5
+ SHA512:
6
+ metadata.gz: 0d99585494a1a67549c8e1a61ac6a2c76c483f6991aaf4c569b3849dbbae984e24abb5c78bd212fc41cfcb084b377625679ae11b446b8eccfba03e5de29c31dd
7
+ data.tar.gz: 535d4d98895e3eae957a7b00aa1dfe11e7c7019db44143eb5642a3bf2baaabf23407b855cc66c776109a85f4d198dcd8971733b4474a3ac9c802b84b3c618e89
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2011 Hiromi Ishii, 2013- github/giraffi
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,164 @@
1
+ require 'time'
2
+ require 'fluent/plugin/input'
3
+ require 'fluent/plugin/parser'
4
+ require 'bunny'
5
+
6
+ module Fluent::Plugin
7
+ ##
8
+ # AMQPInput to be used as a Fluent SOURCE, reading messages from a RabbitMQ
9
+ # message broker
10
+ class AMQPInput < Input
11
+ Fluent::Plugin.register_input('amqp', self)
12
+
13
+ helpers :compat_parameters, :parser
14
+
15
+ # Bunny connection handle
16
+ # - Allows mocking for test purposes
17
+ attr_accessor :connection
18
+
19
+ config_param :tag, :string, default: "hunter.amqp"
20
+
21
+ config_param :host, :string, default: nil
22
+ config_param :hosts, :array, default: nil
23
+ config_param :user, :string, default: "guest"
24
+ config_param :pass, :string, default: "guest", secret: true
25
+ config_param :vhost, :string, default: "/"
26
+ config_param :port, :integer, default: 5672
27
+ config_param :ssl, :bool, default: false
28
+ config_param :verify_ssl, :bool, default: false
29
+ config_param :heartbeat, :integer, default: 60
30
+ config_param :queue, :string, default: nil
31
+ config_param :durable, :bool, default: false
32
+ config_param :exclusive, :bool, default: false
33
+ config_param :auto_delete, :bool, default: false
34
+ config_param :passive, :bool, default: false
35
+ config_param :payload_format, :string, default: "json"
36
+ config_param :tag_key, :bool, default: false
37
+ config_param :tag_header, :string, default: nil
38
+ config_param :time_header, :string, default: nil
39
+ config_param :tls, :bool, default: false
40
+ config_param :tls_cert, :string, default: nil
41
+ config_param :tls_key, :string, default: nil
42
+ config_param :tls_ca_certificates, :array, default: nil
43
+ config_param :tls_verify_peer, :bool, default: true
44
+ config_param :bind_exchange, :bool, default: false
45
+ config_param :exchange, :string, default: ""
46
+ config_param :routing_key, :string, default: "#" # The routing key used to bind queue to exchange - # = matches all, * matches section (tag.*.info)
47
+
48
+ def configure(conf)
49
+ conf['format'] ||= conf['payload_format'] # legacy
50
+ compat_parameters_convert(conf, :parser)
51
+
52
+ super
53
+
54
+ parser_config = conf.elements('parse').first
55
+ if parser_config
56
+ @parser = parser_create(conf: parser_config)
57
+ end
58
+
59
+ @conf = conf
60
+ unless (@host || @hosts) && @queue
61
+ raise Fluent::ConfigError, "'host(s)' and 'queue' must be all specified."
62
+ end
63
+ check_tls_configuration
64
+ end
65
+
66
+ def start
67
+ super
68
+ # Create a new connection, unless its already been provided to us
69
+ @connection = Bunny.new get_connection_options unless @connection
70
+ @connection.start
71
+ @channel = @connection.create_channel
72
+
73
+ if @exclusive && fluentd_worker_id > 0
74
+ log.info 'Config requested exclusive queue with multiple workers'
75
+ @queue += ".#{fluentd_worker_id}"
76
+ log.info "Renamed queue name to include worker id: #{@queue}"
77
+ end
78
+
79
+ q = @channel.queue(@queue, passive: @passive, durable: @durable,
80
+ exclusive: @exclusive, auto_delete: @auto_delete)
81
+ if @bind_exchange
82
+ log.info "Binding #{@queue} to #{@exchange}, :routing_key => #{@routing_key}"
83
+ q.bind(exchange=@exchange, routing_key: @routing_key)
84
+ end
85
+
86
+ q.subscribe do |delivery, meta, msg|
87
+ log.debug "Recieved message #{@msg}"
88
+ payload = parse_payload(msg)
89
+ router.emit(parse_tag(delivery, meta), parse_time(meta), payload)
90
+ end
91
+ end # AMQPInput#run
92
+
93
+ def shutdown
94
+ log.info "Closing connection"
95
+ @connection.stop
96
+ super
97
+ end
98
+
99
+ def multi_workers_ready?
100
+ true
101
+ end
102
+
103
+ private
104
+ def parse_payload(msg)
105
+ if @parser
106
+ parsed = nil
107
+ @parser.parse msg do |_, payload|
108
+ if payload.nil?
109
+ log.warn "failed to parse #{msg}"
110
+ parsed = { "message" => msg }
111
+ else
112
+ parsed = payload
113
+ end
114
+ end
115
+ parsed
116
+ else
117
+ { "message" => msg }
118
+ end
119
+ end
120
+
121
+ def parse_tag( delivery, meta )
122
+ if @tag_key && delivery.routing_key != ''
123
+ delivery.routing_key
124
+ elsif @tag_header && meta[:headers][@tag_header]
125
+ meta[:headers][@tag_header]
126
+ else
127
+ @tag
128
+ end
129
+ end
130
+
131
+ def parse_time( meta )
132
+ if @time_header && meta[:headers][@time_header]
133
+ Fluent::EventTime.from_time(Time.parse( meta[:headers][@time_header] ))
134
+ else
135
+ Fluent::Engine.now
136
+ end
137
+ end
138
+
139
+ def check_tls_configuration()
140
+ if @tls
141
+ unless @tls_key && @tls_cert
142
+ raise Fluent::ConfigError, "'tls_key' and 'tls_cert' must be all specified if tls is enabled."
143
+ end
144
+ end
145
+ end
146
+
147
+ def get_connection_options()
148
+ hosts = @hosts ||= Array.new(1, @host)
149
+ opts = {
150
+ hosts: hosts, port: @port, vhost: @vhost,
151
+ pass: @pass, user: @user, ssl: @ssl,
152
+ verify_ssl: @verify_ssl, heartbeat: @heartbeat,
153
+ tls: @tls,
154
+ tls_cert: @tls_cert,
155
+ tls_key: @tls_key,
156
+ verify_peer: @tls_verify_peer
157
+ }
158
+ opts[:tls_ca_certificates] = @tls_ca_certificates if @tls_ca_certificates
159
+ return opts
160
+ end
161
+
162
+ end # class AMQPInput
163
+
164
+ end # module Fluent::Plugin
@@ -0,0 +1,262 @@
1
+ require 'json'
2
+ require 'fluent/plugin/output'
3
+ require "bunny"
4
+
5
+ module Fluent::Plugin
6
+ ##
7
+ # AMQPOutput to be used as a Fluent MATCHER, sending messages to a RabbitMQ
8
+ # messaging broker
9
+ class AMQPOutput < Output
10
+ Fluent::Plugin.register_output("amqp", self)
11
+
12
+ helpers :compat_parameters
13
+
14
+ DEFAULT_BUFFER_TYPE = "memory"
15
+
16
+ attr_accessor :connection
17
+
18
+ #Attribute readers to support testing
19
+ attr_reader :exch
20
+ attr_reader :channel
21
+
22
+
23
+ config_param :host, :string, default: nil
24
+ config_param :hosts, :array, default: nil
25
+ config_param :user, :string, default: "guest"
26
+ config_param :pass, :string, default: "guest", secret: true
27
+ config_param :vhost, :string, default: "/"
28
+ config_param :port, :integer, default: 5672
29
+ config_param :ssl, :bool, default: false
30
+ config_param :verify_ssl, :bool, default: false
31
+ config_param :heartbeat, :integer, default: 60
32
+ config_param :exchange, :string, default: ""
33
+ config_param :exchange_type, :string, default: "direct"
34
+ config_param :passive, :bool, default: false
35
+ config_param :durable, :bool, default: false
36
+ config_param :auto_delete, :bool, default: false
37
+ config_param :key, :string, default: nil
38
+ config_param :persistent, :bool, default: false
39
+ config_param :tag_key, :bool, default: false
40
+ config_param :tag_header, :string, default: nil
41
+ config_param :time_header, :string, default: nil
42
+ config_param :tls, :bool, default: false
43
+ config_param :tls_cert, :string, default: nil
44
+ config_param :tls_key, :string, default: nil
45
+ config_param :tls_ca_certificates, :array, default: nil
46
+ config_param :tls_verify_peer, :bool, default: true
47
+ config_param :content_type, :string, default: "application/octet"
48
+ config_param :content_encoding, :string, default: nil
49
+
50
+ config_section :header do
51
+ config_set_default :@type, DEFAULT_BUFFER_TYPE
52
+ end
53
+
54
+ config_section :buffer do
55
+ config_set_default :@type, DEFAULT_BUFFER_TYPE
56
+ end
57
+
58
+
59
+ class HeaderElement
60
+ include Fluent::Configurable
61
+
62
+ config_param :name, :string
63
+ config_param :default, :string, default: nil
64
+ config_param :source, default: nil do |val|
65
+ if val.start_with?('[')
66
+ JSON.load(val)
67
+ else
68
+ val.split('.')
69
+ end
70
+ end
71
+
72
+ # Extract a header and value from the input data
73
+ # returning nil if value cannot be derived
74
+ def getValue(data)
75
+ val = getNestedValue(data, @source ) if @source
76
+ val ||= @default if @default
77
+ val
78
+ end
79
+
80
+ def getNestedValue(data, path)
81
+ temp_data = data
82
+ temp_path = path.dup
83
+ until temp_data.nil? or temp_path.empty?
84
+ temp_data = temp_data[temp_path.shift]
85
+ end
86
+ temp_data
87
+ end
88
+ end
89
+
90
+ def configure(conf)
91
+ compat_parameters_convert(conf, :buffer)
92
+ super
93
+ @conf = conf
94
+
95
+ # Extract the header configuration into a collection
96
+ @headers = conf.elements.select {|e|
97
+ e.name == 'header'
98
+ }.map {|e|
99
+ he = HeaderElement.new
100
+ he.configure(e)
101
+ unless he.source || he.default
102
+ raise Fluent::ConfigError, "At least 'default' or 'source' must must be defined in a header configuration section."
103
+ end
104
+ he
105
+ }
106
+
107
+ unless @host || @hosts
108
+ raise Fluent::ConfigError, "'host' or 'hosts' must be specified."
109
+ end
110
+ unless @key || @tag_key
111
+ raise Fluent::ConfigError, "Either 'key' or 'tag_key' must be set."
112
+ end
113
+ check_tls_configuration
114
+ end
115
+
116
+ def start
117
+ super
118
+ begin
119
+ log.info "Connecting to RabbitMQ..."
120
+ @connection = Bunny.new(get_connection_options) unless @connection
121
+ @connection.start
122
+ rescue Bunny::TCPConnectionFailed => e
123
+ log.error "Connection to #{@host} failed"
124
+ rescue Bunny::PossibleAuthenticationFailureError => e
125
+ log.error "Could not authenticate as #{@user}"
126
+ end
127
+
128
+ log.info 'Creating new channel'
129
+ @channel = @connection.create_channel
130
+
131
+ return if @exchange.to_s =~ CHUNK_KEY_PLACEHOLDER_PATTERN
132
+
133
+ log.info 'Creating new exchange (in start)', exchange: @exchange
134
+ @exch = @channel.exchange(@exchange, type: @exchange_type.intern,
135
+ passive: @passive, durable: @durable,
136
+ auto_delete: @auto_delete)
137
+ end
138
+
139
+ def shutdown
140
+ @connection.stop
141
+ super
142
+ end
143
+
144
+ def multi_workers_ready?
145
+ true
146
+ end
147
+
148
+ def formatted_to_msgpack_binary
149
+ true
150
+ end
151
+
152
+ def format(tag, time, record)
153
+ [tag, time, record].to_msgpack
154
+ end
155
+
156
+ def write(chunk)
157
+ begin
158
+ log.debug 'in write', chunk_id: dump_unique_id_hex(chunk.unique_id)
159
+ log.debug 'raw exchange value is', exchange: @exchange.to_s
160
+
161
+ if @exchange.to_s =~ CHUNK_KEY_PLACEHOLDER_PATTERN
162
+ exchange_name = extract_placeholders(@exchange, chunk)
163
+ log.info 'resolved exchange value is', exchange_name: exchange_name
164
+ @exch = @channel.exchange(exchange_name, type: @exchange_type.intern,
165
+ passive: @passive, durable: @durable,
166
+ auto_delete: @auto_delete)
167
+ end
168
+
169
+ log.debug 'writing data to exchange', chunk_id: dump_unique_id_hex(chunk.unique_id)
170
+
171
+ chunk.msgpack_each do |(tag, time, data)|
172
+ begin
173
+ msg_headers = headers(tag,time,data)
174
+
175
+ begin
176
+ data = JSON.dump( data ) unless data.is_a?( String )
177
+ rescue JSON::GeneratorError => e
178
+ log.warn "Failure converting data object to json string: #{e.message} - sending as raw object"
179
+ # Debug only - otherwise we may pollute the fluent logs with unparseable events and loop
180
+ log.debug "JSON.dump failure converting [#{data}]"
181
+ end
182
+
183
+ log.info "Sending message #{data}, :key => #{routing_key( tag)} :headers => #{headers(tag,time,data)}"
184
+ @exch.publish(
185
+ data,
186
+ key: routing_key( tag ),
187
+ persistent: @persistent,
188
+ headers: msg_headers,
189
+ content_type: @content_type,
190
+ content_encoding: @content_encoding)
191
+
192
+ # :nocov:
193
+ # Hard to throw StandardError through test code
194
+ rescue StandardError => e
195
+ # This protects against invalid byteranges and other errors at a per-message level
196
+ log.error "Unexpected error during message publishing: #{e.message}"
197
+ log.debug "Failure in publishing message [#{data}]"
198
+ end
199
+ end
200
+ rescue MessagePack::MalformedFormatError => e
201
+ # This has been observed when a server has filled the partition containing
202
+ # the buffer files, and during replay the chunks were malformed
203
+ log.error "Malformed msgpack in chunk - Did your server run out of space during buffering? #{e.message}"
204
+ rescue StandardError => e
205
+ # Just in case theres any other errors during chunk loading.
206
+ log.error "Unexpected error during message publishing: #{e.message}"
207
+ end
208
+ # :nocov:
209
+ end
210
+
211
+
212
+ def routing_key( tag )
213
+ if @tag_key
214
+ tag
215
+ else
216
+ @key
217
+ end
218
+ end
219
+
220
+ def headers( tag, time, data )
221
+ h = {}
222
+
223
+ log.debug "Processing Headers: #{@headers}"
224
+ # A little messy this...
225
+ # Trying to allow for header overrides where a header defined
226
+ # earlier will be used if a later header is returning nil (ie not found and no default)
227
+ h = Hash[ @headers
228
+ .collect{|v| [v.name, v.getValue(data) ]}
229
+ .delete_if{|x| x.last.nil?}
230
+ ]
231
+
232
+ h[@tag_header] = tag if @tag_header
233
+ h[@time_header] = Time.at(time).utc.to_s if @time_header
234
+
235
+ h
236
+ end
237
+
238
+
239
+ def check_tls_configuration()
240
+ if @tls
241
+ unless @tls_key && @tls_cert
242
+ raise Fluent::ConfigError, "'tls_key' and 'tls_cert' must be all specified if tls is enabled."
243
+ end
244
+ end
245
+ end
246
+
247
+ def get_connection_options()
248
+ hosts = @hosts ||= Array.new(1, @host)
249
+ opts = {
250
+ hosts: hosts, port: @port, vhost: @vhost,
251
+ pass: @pass, user: @user, ssl: @ssl,
252
+ verify_ssl: @verify_ssl, heartbeat: @heartbeat,
253
+ tls: @tls || nil,
254
+ tls_cert: @tls_cert,
255
+ tls_key: @tls_key,
256
+ verify_peer: @tls_verify_peer
257
+ }
258
+ opts[:tls_ca_certificates] = @tls_ca_certificates if @tls_ca_certificates
259
+ return opts
260
+ end
261
+ end
262
+ end
metadata ADDED
@@ -0,0 +1,174 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-amqp-sboagibm
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.21.0
5
+ platform: ruby
6
+ authors:
7
+ - Hiromi Ishii
8
+ - Team Giraffi
9
+ - HiganWorks LLC
10
+ - Toby Jackson
11
+ autorequire:
12
+ bindir: bin
13
+ cert_chain: []
14
+ date: 2018-11-16 00:00:00.000000000 Z
15
+ dependencies:
16
+ - !ruby/object:Gem::Dependency
17
+ name: fluentd
18
+ requirement: !ruby/object:Gem::Requirement
19
+ requirements:
20
+ - - ">="
21
+ - !ruby/object:Gem::Version
22
+ version: 0.14.8
23
+ - - "<"
24
+ - !ruby/object:Gem::Version
25
+ version: '2'
26
+ type: :runtime
27
+ prerelease: false
28
+ version_requirements: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - ">="
31
+ - !ruby/object:Gem::Version
32
+ version: 0.14.8
33
+ - - "<"
34
+ - !ruby/object:Gem::Version
35
+ version: '2'
36
+ - !ruby/object:Gem::Dependency
37
+ name: bunny
38
+ requirement: !ruby/object:Gem::Requirement
39
+ requirements:
40
+ - - ">="
41
+ - !ruby/object:Gem::Version
42
+ version: '1.7'
43
+ - - "<"
44
+ - !ruby/object:Gem::Version
45
+ version: '3'
46
+ type: :runtime
47
+ prerelease: false
48
+ version_requirements: !ruby/object:Gem::Requirement
49
+ requirements:
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: '1.7'
53
+ - - "<"
54
+ - !ruby/object:Gem::Version
55
+ version: '3'
56
+ - !ruby/object:Gem::Dependency
57
+ name: bunny-mock
58
+ requirement: !ruby/object:Gem::Requirement
59
+ requirements:
60
+ - - ">="
61
+ - !ruby/object:Gem::Version
62
+ version: '1.0'
63
+ type: :development
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - ">="
68
+ - !ruby/object:Gem::Version
69
+ version: '1.0'
70
+ - !ruby/object:Gem::Dependency
71
+ name: shoulda
72
+ requirement: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - ">="
75
+ - !ruby/object:Gem::Version
76
+ version: 3.5.0
77
+ type: :development
78
+ prerelease: false
79
+ version_requirements: !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - ">="
82
+ - !ruby/object:Gem::Version
83
+ version: 3.5.0
84
+ - !ruby/object:Gem::Dependency
85
+ name: rake
86
+ requirement: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - ">="
89
+ - !ruby/object:Gem::Version
90
+ version: '0'
91
+ type: :development
92
+ prerelease: false
93
+ version_requirements: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - ">="
96
+ - !ruby/object:Gem::Version
97
+ version: '0'
98
+ - !ruby/object:Gem::Dependency
99
+ name: minitest
100
+ requirement: !ruby/object:Gem::Requirement
101
+ requirements:
102
+ - - "<"
103
+ - !ruby/object:Gem::Version
104
+ version: 5.0.0
105
+ type: :development
106
+ prerelease: false
107
+ version_requirements: !ruby/object:Gem::Requirement
108
+ requirements:
109
+ - - "<"
110
+ - !ruby/object:Gem::Version
111
+ version: 5.0.0
112
+ - !ruby/object:Gem::Dependency
113
+ name: test-unit
114
+ requirement: !ruby/object:Gem::Requirement
115
+ requirements:
116
+ - - ">="
117
+ - !ruby/object:Gem::Version
118
+ version: 3.1.0
119
+ type: :development
120
+ prerelease: false
121
+ version_requirements: !ruby/object:Gem::Requirement
122
+ requirements:
123
+ - - ">="
124
+ - !ruby/object:Gem::Version
125
+ version: 3.1.0
126
+ - !ruby/object:Gem::Dependency
127
+ name: simplecov
128
+ requirement: !ruby/object:Gem::Requirement
129
+ requirements:
130
+ - - ">="
131
+ - !ruby/object:Gem::Version
132
+ version: '0.10'
133
+ type: :development
134
+ prerelease: false
135
+ version_requirements: !ruby/object:Gem::Requirement
136
+ requirements:
137
+ - - ">="
138
+ - !ruby/object:Gem::Version
139
+ version: '0.10'
140
+ description: AMQP input/output plugin for fluentd
141
+ email: sawanoboriyu@higanworks.com
142
+ executables: []
143
+ extensions: []
144
+ extra_rdoc_files:
145
+ - LICENSE.txt
146
+ files:
147
+ - LICENSE.txt
148
+ - lib/fluent/plugin/in_amqp.rb
149
+ - lib/fluent/plugin/out_amqp.rb
150
+ homepage: https://github.com/sboagibm/fluent-plugin-amqp
151
+ licenses:
152
+ - Apache License, Version 2.0
153
+ metadata: {}
154
+ post_install_message:
155
+ rdoc_options: []
156
+ require_paths:
157
+ - lib
158
+ required_ruby_version: !ruby/object:Gem::Requirement
159
+ requirements:
160
+ - - ">="
161
+ - !ruby/object:Gem::Version
162
+ version: 2.1.0
163
+ required_rubygems_version: !ruby/object:Gem::Requirement
164
+ requirements:
165
+ - - ">="
166
+ - !ruby/object:Gem::Version
167
+ version: '0'
168
+ requirements: []
169
+ rubyforge_project:
170
+ rubygems_version: 2.5.2.3
171
+ signing_key:
172
+ specification_version: 4
173
+ summary: AMQP input/output plugin or fluentd
174
+ test_files: []