fluent-plugin-amqp 0.10.0 → 0.14.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/lib/fluent/plugin/in_amqp.rb +32 -19
- data/lib/fluent/plugin/out_amqp.rb +94 -23
- metadata +8 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 83133837639cc240c17c319ddc075ff365e4ed599e393439216f00a7ba62ec12
|
4
|
+
data.tar.gz: 51005871d326dd9ee209f77e96c602892e2ef8bda5a126753007bd5e82770470
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8bd4d7ca5e802b74fcee42bd7050dc63e20d4ee0fdbbf15ab8b3084df3c302ec674b1452ada1ff19ba48a0d4c59affc191f71bd93f35277b115d0aff42fef228
|
7
|
+
data.tar.gz: 88494f525f1b9c6d3b211068e1a3a5ee04b4dc03a34dded4415534046e6329fe9ad480e04cc12d4ea08104c3837d8f2ae7c4f70d0fd09d64a76b1f5930a70b6d
|
@@ -44,6 +44,8 @@ module Fluent::Plugin
|
|
44
44
|
config_param :bind_exchange, :bool, default: false
|
45
45
|
config_param :exchange, :string, default: ""
|
46
46
|
config_param :routing_key, :string, default: "#" # The routing key used to bind queue to exchange - # = matches all, * matches section (tag.*.info)
|
47
|
+
config_param :include_headers, :bool, default: false
|
48
|
+
config_param :auth_mechanism, :string, default: nil
|
47
49
|
|
48
50
|
def configure(conf)
|
49
51
|
conf['format'] ||= conf['payload_format'] # legacy
|
@@ -60,7 +62,6 @@ module Fluent::Plugin
|
|
60
62
|
unless (@host || @hosts) && @queue
|
61
63
|
raise Fluent::ConfigError, "'host(s)' and 'queue' must be all specified."
|
62
64
|
end
|
63
|
-
check_tls_configuration
|
64
65
|
end
|
65
66
|
|
66
67
|
def start
|
@@ -69,6 +70,13 @@ module Fluent::Plugin
|
|
69
70
|
@connection = Bunny.new get_connection_options unless @connection
|
70
71
|
@connection.start
|
71
72
|
@channel = @connection.create_channel
|
73
|
+
|
74
|
+
if @exclusive && fluentd_worker_id > 0
|
75
|
+
log.info 'Config requested exclusive queue with multiple workers'
|
76
|
+
@queue += ".#{fluentd_worker_id}"
|
77
|
+
log.info "Renamed queue name to include worker id: #{@queue}"
|
78
|
+
end
|
79
|
+
|
72
80
|
q = @channel.queue(@queue, passive: @passive, durable: @durable,
|
73
81
|
exclusive: @exclusive, auto_delete: @auto_delete)
|
74
82
|
if @bind_exchange
|
@@ -77,8 +85,10 @@ module Fluent::Plugin
|
|
77
85
|
end
|
78
86
|
|
79
87
|
q.subscribe do |delivery, meta, msg|
|
80
|
-
log.debug "Recieved message #{
|
81
|
-
|
88
|
+
log.debug "Recieved message #{msg}"
|
89
|
+
log.debug "Recieved message MetaData #{meta}"
|
90
|
+
payload = parse_payload(msg, meta)
|
91
|
+
log.debug "Parsed Payload #{payload}"
|
82
92
|
router.emit(parse_tag(delivery, meta), parse_time(meta), payload)
|
83
93
|
end
|
84
94
|
end # AMQPInput#run
|
@@ -89,21 +99,30 @@ module Fluent::Plugin
|
|
89
99
|
super
|
90
100
|
end
|
91
101
|
|
102
|
+
def multi_workers_ready?
|
103
|
+
true
|
104
|
+
end
|
105
|
+
|
92
106
|
private
|
93
|
-
def parse_payload(msg)
|
107
|
+
def parse_payload(msg, meta)
|
108
|
+
parsed = nil
|
94
109
|
if @parser
|
95
|
-
parsed = nil
|
96
110
|
@parser.parse msg do |_, payload|
|
97
111
|
if payload.nil?
|
98
112
|
log.warn "failed to parse #{msg}"
|
99
|
-
parsed = {
|
113
|
+
parsed = { 'message' => msg }
|
100
114
|
else
|
101
115
|
parsed = payload
|
102
116
|
end
|
103
117
|
end
|
104
|
-
parsed
|
105
118
|
else
|
106
|
-
{
|
119
|
+
parsed = { 'message' => msg }
|
120
|
+
end
|
121
|
+
if @include_headers
|
122
|
+
log.debug 'Adding headers'
|
123
|
+
{ 'headers' => meta[:headers] }.merge(parsed)
|
124
|
+
else
|
125
|
+
parsed
|
107
126
|
end
|
108
127
|
end
|
109
128
|
|
@@ -125,14 +144,6 @@ module Fluent::Plugin
|
|
125
144
|
end
|
126
145
|
end
|
127
146
|
|
128
|
-
def check_tls_configuration()
|
129
|
-
if @tls
|
130
|
-
unless @tls_key && @tls_cert
|
131
|
-
raise Fluent::ConfigError, "'tls_key' and 'tls_cert' must be all specified if tls is enabled."
|
132
|
-
end
|
133
|
-
end
|
134
|
-
end
|
135
|
-
|
136
147
|
def get_connection_options()
|
137
148
|
hosts = @hosts ||= Array.new(1, @host)
|
138
149
|
opts = {
|
@@ -140,10 +151,12 @@ module Fluent::Plugin
|
|
140
151
|
pass: @pass, user: @user, ssl: @ssl,
|
141
152
|
verify_ssl: @verify_ssl, heartbeat: @heartbeat,
|
142
153
|
tls: @tls,
|
143
|
-
|
144
|
-
|
145
|
-
verify_peer: @tls_verify_peer
|
154
|
+
verify_peer: @tls_verify_peer,
|
155
|
+
auth_mechanism: @auth_mechanism
|
146
156
|
}
|
157
|
+
# Include additional optional TLS configurations
|
158
|
+
opts[:tls_key] = @tls_key if @tls_key
|
159
|
+
opts[:tls_cert] = @tls_cert if @tls_cert
|
147
160
|
opts[:tls_ca_certificates] = @tls_ca_certificates if @tls_ca_certificates
|
148
161
|
return opts
|
149
162
|
end
|
@@ -44,22 +44,73 @@ module Fluent::Plugin
|
|
44
44
|
config_param :tls_key, :string, default: nil
|
45
45
|
config_param :tls_ca_certificates, :array, default: nil
|
46
46
|
config_param :tls_verify_peer, :bool, default: true
|
47
|
+
config_param :content_type, :string, default: "application/octet"
|
48
|
+
config_param :content_encoding, :string, default: nil
|
49
|
+
config_param :auth_mechanism, :string, default: nil
|
50
|
+
|
51
|
+
config_section :header do
|
52
|
+
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
53
|
+
end
|
47
54
|
|
48
55
|
config_section :buffer do
|
49
56
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
50
57
|
end
|
51
58
|
|
59
|
+
|
60
|
+
class HeaderElement
|
61
|
+
include Fluent::Configurable
|
62
|
+
|
63
|
+
config_param :name, :string
|
64
|
+
config_param :default, :string, default: nil
|
65
|
+
config_param :source, default: nil do |val|
|
66
|
+
if val.start_with?('[')
|
67
|
+
JSON.load(val)
|
68
|
+
else
|
69
|
+
val.split('.')
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
# Extract a header and value from the input data
|
74
|
+
# returning nil if value cannot be derived
|
75
|
+
def getValue(data)
|
76
|
+
val = getNestedValue(data, @source ) if @source
|
77
|
+
val ||= @default if @default
|
78
|
+
val
|
79
|
+
end
|
80
|
+
|
81
|
+
def getNestedValue(data, path)
|
82
|
+
temp_data = data
|
83
|
+
temp_path = path.dup
|
84
|
+
until temp_data.nil? or temp_path.empty?
|
85
|
+
temp_data = temp_data[temp_path.shift]
|
86
|
+
end
|
87
|
+
temp_data
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
52
91
|
def configure(conf)
|
53
92
|
compat_parameters_convert(conf, :buffer)
|
54
93
|
super
|
55
94
|
@conf = conf
|
95
|
+
|
96
|
+
# Extract the header configuration into a collection
|
97
|
+
@headers = conf.elements.select {|e|
|
98
|
+
e.name == 'header'
|
99
|
+
}.map {|e|
|
100
|
+
he = HeaderElement.new
|
101
|
+
he.configure(e)
|
102
|
+
unless he.source || he.default
|
103
|
+
raise Fluent::ConfigError, "At least 'default' or 'source' must must be defined in a header configuration section."
|
104
|
+
end
|
105
|
+
he
|
106
|
+
}
|
107
|
+
|
56
108
|
unless @host || @hosts
|
57
109
|
raise Fluent::ConfigError, "'host' or 'hosts' must be specified."
|
58
110
|
end
|
59
111
|
unless @key || @tag_key
|
60
112
|
raise Fluent::ConfigError, "Either 'key' or 'tag_key' must be set."
|
61
113
|
end
|
62
|
-
check_tls_configuration
|
63
114
|
end
|
64
115
|
|
65
116
|
def start
|
@@ -86,6 +137,10 @@ module Fluent::Plugin
|
|
86
137
|
super
|
87
138
|
end
|
88
139
|
|
140
|
+
def multi_workers_ready?
|
141
|
+
true
|
142
|
+
end
|
143
|
+
|
89
144
|
def formatted_to_msgpack_binary
|
90
145
|
true
|
91
146
|
end
|
@@ -98,13 +153,27 @@ module Fluent::Plugin
|
|
98
153
|
begin
|
99
154
|
chunk.msgpack_each do |(tag, time, data)|
|
100
155
|
begin
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
156
|
+
msg_headers = headers(tag,time,data)
|
157
|
+
|
158
|
+
begin
|
159
|
+
data = JSON.dump( data ) unless data.is_a?( String )
|
160
|
+
rescue JSON::GeneratorError => e
|
161
|
+
log.warn "Failure converting data object to json string: #{e.message} - sending as raw object"
|
162
|
+
# Debug only - otherwise we may pollute the fluent logs with unparseable events and loop
|
163
|
+
log.debug "JSON.dump failure converting [#{data}]"
|
164
|
+
end
|
165
|
+
|
166
|
+
log.debug "Sending message #{data}, :key => #{routing_key( tag)} :headers => #{headers(tag,time,data)}"
|
167
|
+
@exch.publish(
|
168
|
+
data,
|
169
|
+
key: routing_key( tag ),
|
170
|
+
persistent: @persistent,
|
171
|
+
headers: msg_headers,
|
172
|
+
content_type: @content_type,
|
173
|
+
content_encoding: @content_encoding)
|
174
|
+
|
175
|
+
# :nocov:
|
176
|
+
# Hard to throw StandardError through test code
|
108
177
|
rescue StandardError => e
|
109
178
|
# This protects against invalid byteranges and other errors at a per-message level
|
110
179
|
log.error "Unexpected error during message publishing: #{e.message}"
|
@@ -119,6 +188,7 @@ module Fluent::Plugin
|
|
119
188
|
# Just in case theres any other errors during chunk loading.
|
120
189
|
log.error "Unexpected error during message publishing: #{e.message}"
|
121
190
|
end
|
191
|
+
# :nocov:
|
122
192
|
end
|
123
193
|
|
124
194
|
|
@@ -130,21 +200,22 @@ module Fluent::Plugin
|
|
130
200
|
end
|
131
201
|
end
|
132
202
|
|
133
|
-
def headers( tag, time )
|
134
|
-
{}
|
135
|
-
h[@tag_header] = tag if @tag_header
|
136
|
-
h[@time_header] = Time.at(time).utc.to_s if @time_header
|
137
|
-
end
|
138
|
-
end
|
203
|
+
def headers( tag, time, data )
|
204
|
+
h = {}
|
139
205
|
|
206
|
+
log.debug "Processing Headers: #{@headers}"
|
207
|
+
# A little messy this...
|
208
|
+
# Trying to allow for header overrides where a header defined
|
209
|
+
# earlier will be used if a later header is returning nil (ie not found and no default)
|
210
|
+
h = Hash[ @headers
|
211
|
+
.collect{|v| [v.name, v.getValue(data) ]}
|
212
|
+
.delete_if{|x| x.last.nil?}
|
213
|
+
]
|
140
214
|
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
raise Fluent::ConfigError, "'tls_key' and 'tls_cert' must be all specified if tls is enabled."
|
146
|
-
end
|
147
|
-
end
|
215
|
+
h[@tag_header] = tag if @tag_header
|
216
|
+
h[@time_header] = Time.at(time).utc.to_s if @time_header
|
217
|
+
|
218
|
+
h
|
148
219
|
end
|
149
220
|
|
150
221
|
def get_connection_options()
|
@@ -156,11 +227,11 @@ module Fluent::Plugin
|
|
156
227
|
tls: @tls || nil,
|
157
228
|
tls_cert: @tls_cert,
|
158
229
|
tls_key: @tls_key,
|
159
|
-
verify_peer: @tls_verify_peer
|
230
|
+
verify_peer: @tls_verify_peer,
|
231
|
+
auth_mechanism: @auth_mechanism
|
160
232
|
}
|
161
233
|
opts[:tls_ca_certificates] = @tls_ca_certificates if @tls_ca_certificates
|
162
234
|
return opts
|
163
235
|
end
|
164
|
-
|
165
236
|
end
|
166
237
|
end
|
metadata
CHANGED
@@ -1,17 +1,17 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-amqp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.14.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hiromi Ishii
|
8
8
|
- Team Giraffi
|
9
9
|
- HiganWorks LLC
|
10
10
|
- Toby Jackson
|
11
|
-
autorequire:
|
11
|
+
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date:
|
14
|
+
date: 2020-07-12 00:00:00.000000000 Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: fluentd
|
@@ -151,7 +151,7 @@ homepage: http://github.com/giraffi/fluent-plugin-amqp
|
|
151
151
|
licenses:
|
152
152
|
- Apache License, Version 2.0
|
153
153
|
metadata: {}
|
154
|
-
post_install_message:
|
154
|
+
post_install_message:
|
155
155
|
rdoc_options: []
|
156
156
|
require_paths:
|
157
157
|
- lib
|
@@ -159,16 +159,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
159
159
|
requirements:
|
160
160
|
- - ">="
|
161
161
|
- !ruby/object:Gem::Version
|
162
|
-
version:
|
162
|
+
version: 2.1.0
|
163
163
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
164
164
|
requirements:
|
165
165
|
- - ">="
|
166
166
|
- !ruby/object:Gem::Version
|
167
|
-
version:
|
167
|
+
version: '0'
|
168
168
|
requirements: []
|
169
|
-
|
170
|
-
|
171
|
-
signing_key:
|
169
|
+
rubygems_version: 3.1.2
|
170
|
+
signing_key:
|
172
171
|
specification_version: 4
|
173
172
|
summary: AMQP input/output plugin or fluentd
|
174
173
|
test_files: []
|