fluent-plugin-amqp 0.10.0 → 0.14.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: b4afb9f6144c0448aefb570fbe50d47f8ad4845c
4
- data.tar.gz: 9c07383eb91e5e825d12fb23a93b294245afb9c8
2
+ SHA256:
3
+ metadata.gz: 83133837639cc240c17c319ddc075ff365e4ed599e393439216f00a7ba62ec12
4
+ data.tar.gz: 51005871d326dd9ee209f77e96c602892e2ef8bda5a126753007bd5e82770470
5
5
  SHA512:
6
- metadata.gz: dc9872fd8d8fef5e4f38bebacbf440a7b6b89568f4285592d09190d4ecdf40ec32f9f94606f6f273a7e6ca1b2737d53d0aafc237a9b5002c62f41da07e006ab7
7
- data.tar.gz: 972dcc4296cc3e0c87245fde0763336a39e8e436237ec3f7930516e68ca0db9c13aa7e19714ef19549d8770192c595960737c242a0c13da38b759effcbd8d48c
6
+ metadata.gz: 8bd4d7ca5e802b74fcee42bd7050dc63e20d4ee0fdbbf15ab8b3084df3c302ec674b1452ada1ff19ba48a0d4c59affc191f71bd93f35277b115d0aff42fef228
7
+ data.tar.gz: 88494f525f1b9c6d3b211068e1a3a5ee04b4dc03a34dded4415534046e6329fe9ad480e04cc12d4ea08104c3837d8f2ae7c4f70d0fd09d64a76b1f5930a70b6d
@@ -44,6 +44,8 @@ module Fluent::Plugin
44
44
  config_param :bind_exchange, :bool, default: false
45
45
  config_param :exchange, :string, default: ""
46
46
  config_param :routing_key, :string, default: "#" # The routing key used to bind queue to exchange - # = matches all, * matches section (tag.*.info)
47
+ config_param :include_headers, :bool, default: false
48
+ config_param :auth_mechanism, :string, default: nil
47
49
 
48
50
  def configure(conf)
49
51
  conf['format'] ||= conf['payload_format'] # legacy
@@ -60,7 +62,6 @@ module Fluent::Plugin
60
62
  unless (@host || @hosts) && @queue
61
63
  raise Fluent::ConfigError, "'host(s)' and 'queue' must be all specified."
62
64
  end
63
- check_tls_configuration
64
65
  end
65
66
 
66
67
  def start
@@ -69,6 +70,13 @@ module Fluent::Plugin
69
70
  @connection = Bunny.new get_connection_options unless @connection
70
71
  @connection.start
71
72
  @channel = @connection.create_channel
73
+
74
+ if @exclusive && fluentd_worker_id > 0
75
+ log.info 'Config requested exclusive queue with multiple workers'
76
+ @queue += ".#{fluentd_worker_id}"
77
+ log.info "Renamed queue name to include worker id: #{@queue}"
78
+ end
79
+
72
80
  q = @channel.queue(@queue, passive: @passive, durable: @durable,
73
81
  exclusive: @exclusive, auto_delete: @auto_delete)
74
82
  if @bind_exchange
@@ -77,8 +85,10 @@ module Fluent::Plugin
77
85
  end
78
86
 
79
87
  q.subscribe do |delivery, meta, msg|
80
- log.debug "Recieved message #{@msg}"
81
- payload = parse_payload(msg)
88
+ log.debug "Recieved message #{msg}"
89
+ log.debug "Recieved message MetaData #{meta}"
90
+ payload = parse_payload(msg, meta)
91
+ log.debug "Parsed Payload #{payload}"
82
92
  router.emit(parse_tag(delivery, meta), parse_time(meta), payload)
83
93
  end
84
94
  end # AMQPInput#run
@@ -89,21 +99,30 @@ module Fluent::Plugin
89
99
  super
90
100
  end
91
101
 
102
+ def multi_workers_ready?
103
+ true
104
+ end
105
+
92
106
  private
93
- def parse_payload(msg)
107
+ def parse_payload(msg, meta)
108
+ parsed = nil
94
109
  if @parser
95
- parsed = nil
96
110
  @parser.parse msg do |_, payload|
97
111
  if payload.nil?
98
112
  log.warn "failed to parse #{msg}"
99
- parsed = { "message" => msg }
113
+ parsed = { 'message' => msg }
100
114
  else
101
115
  parsed = payload
102
116
  end
103
117
  end
104
- parsed
105
118
  else
106
- { "message" => msg }
119
+ parsed = { 'message' => msg }
120
+ end
121
+ if @include_headers
122
+ log.debug 'Adding headers'
123
+ { 'headers' => meta[:headers] }.merge(parsed)
124
+ else
125
+ parsed
107
126
  end
108
127
  end
109
128
 
@@ -125,14 +144,6 @@ module Fluent::Plugin
125
144
  end
126
145
  end
127
146
 
128
- def check_tls_configuration()
129
- if @tls
130
- unless @tls_key && @tls_cert
131
- raise Fluent::ConfigError, "'tls_key' and 'tls_cert' must be all specified if tls is enabled."
132
- end
133
- end
134
- end
135
-
136
147
  def get_connection_options()
137
148
  hosts = @hosts ||= Array.new(1, @host)
138
149
  opts = {
@@ -140,10 +151,12 @@ module Fluent::Plugin
140
151
  pass: @pass, user: @user, ssl: @ssl,
141
152
  verify_ssl: @verify_ssl, heartbeat: @heartbeat,
142
153
  tls: @tls,
143
- tls_cert: @tls_cert,
144
- tls_key: @tls_key,
145
- verify_peer: @tls_verify_peer
154
+ verify_peer: @tls_verify_peer,
155
+ auth_mechanism: @auth_mechanism
146
156
  }
157
+ # Include additional optional TLS configurations
158
+ opts[:tls_key] = @tls_key if @tls_key
159
+ opts[:tls_cert] = @tls_cert if @tls_cert
147
160
  opts[:tls_ca_certificates] = @tls_ca_certificates if @tls_ca_certificates
148
161
  return opts
149
162
  end
@@ -44,22 +44,73 @@ module Fluent::Plugin
44
44
  config_param :tls_key, :string, default: nil
45
45
  config_param :tls_ca_certificates, :array, default: nil
46
46
  config_param :tls_verify_peer, :bool, default: true
47
+ config_param :content_type, :string, default: "application/octet"
48
+ config_param :content_encoding, :string, default: nil
49
+ config_param :auth_mechanism, :string, default: nil
50
+
51
+ config_section :header do
52
+ config_set_default :@type, DEFAULT_BUFFER_TYPE
53
+ end
47
54
 
48
55
  config_section :buffer do
49
56
  config_set_default :@type, DEFAULT_BUFFER_TYPE
50
57
  end
51
58
 
59
+
60
+ class HeaderElement
61
+ include Fluent::Configurable
62
+
63
+ config_param :name, :string
64
+ config_param :default, :string, default: nil
65
+ config_param :source, default: nil do |val|
66
+ if val.start_with?('[')
67
+ JSON.load(val)
68
+ else
69
+ val.split('.')
70
+ end
71
+ end
72
+
73
+ # Extract a header and value from the input data
74
+ # returning nil if value cannot be derived
75
+ def getValue(data)
76
+ val = getNestedValue(data, @source ) if @source
77
+ val ||= @default if @default
78
+ val
79
+ end
80
+
81
+ def getNestedValue(data, path)
82
+ temp_data = data
83
+ temp_path = path.dup
84
+ until temp_data.nil? or temp_path.empty?
85
+ temp_data = temp_data[temp_path.shift]
86
+ end
87
+ temp_data
88
+ end
89
+ end
90
+
52
91
  def configure(conf)
53
92
  compat_parameters_convert(conf, :buffer)
54
93
  super
55
94
  @conf = conf
95
+
96
+ # Extract the header configuration into a collection
97
+ @headers = conf.elements.select {|e|
98
+ e.name == 'header'
99
+ }.map {|e|
100
+ he = HeaderElement.new
101
+ he.configure(e)
102
+ unless he.source || he.default
103
+ raise Fluent::ConfigError, "At least 'default' or 'source' must must be defined in a header configuration section."
104
+ end
105
+ he
106
+ }
107
+
56
108
  unless @host || @hosts
57
109
  raise Fluent::ConfigError, "'host' or 'hosts' must be specified."
58
110
  end
59
111
  unless @key || @tag_key
60
112
  raise Fluent::ConfigError, "Either 'key' or 'tag_key' must be set."
61
113
  end
62
- check_tls_configuration
63
114
  end
64
115
 
65
116
  def start
@@ -86,6 +137,10 @@ module Fluent::Plugin
86
137
  super
87
138
  end
88
139
 
140
+ def multi_workers_ready?
141
+ true
142
+ end
143
+
89
144
  def formatted_to_msgpack_binary
90
145
  true
91
146
  end
@@ -98,13 +153,27 @@ module Fluent::Plugin
98
153
  begin
99
154
  chunk.msgpack_each do |(tag, time, data)|
100
155
  begin
101
- data = JSON.dump( data ) unless data.is_a?( String )
102
- log.debug "Sending message #{data}, :key => #{routing_key( tag)} :headers => #{headers(tag,time)}"
103
- @exch.publish(data, key: routing_key( tag ), persistent: @persistent, headers: headers( tag, time ))
104
- rescue JSON::GeneratorError => e
105
- log.error "Failure converting data object to json string: #{e.message}"
106
- # Debug only - otherwise we may pollute the fluent logs with unparseable events and loop
107
- log.debug "JSON.dump failure converting [#{data}]"
156
+ msg_headers = headers(tag,time,data)
157
+
158
+ begin
159
+ data = JSON.dump( data ) unless data.is_a?( String )
160
+ rescue JSON::GeneratorError => e
161
+ log.warn "Failure converting data object to json string: #{e.message} - sending as raw object"
162
+ # Debug only - otherwise we may pollute the fluent logs with unparseable events and loop
163
+ log.debug "JSON.dump failure converting [#{data}]"
164
+ end
165
+
166
+ log.debug "Sending message #{data}, :key => #{routing_key( tag)} :headers => #{headers(tag,time,data)}"
167
+ @exch.publish(
168
+ data,
169
+ key: routing_key( tag ),
170
+ persistent: @persistent,
171
+ headers: msg_headers,
172
+ content_type: @content_type,
173
+ content_encoding: @content_encoding)
174
+
175
+ # :nocov:
176
+ # Hard to throw StandardError through test code
108
177
  rescue StandardError => e
109
178
  # This protects against invalid byteranges and other errors at a per-message level
110
179
  log.error "Unexpected error during message publishing: #{e.message}"
@@ -119,6 +188,7 @@ module Fluent::Plugin
119
188
  # Just in case theres any other errors during chunk loading.
120
189
  log.error "Unexpected error during message publishing: #{e.message}"
121
190
  end
191
+ # :nocov:
122
192
  end
123
193
 
124
194
 
@@ -130,21 +200,22 @@ module Fluent::Plugin
130
200
  end
131
201
  end
132
202
 
133
- def headers( tag, time )
134
- {}.tap do |h|
135
- h[@tag_header] = tag if @tag_header
136
- h[@time_header] = Time.at(time).utc.to_s if @time_header
137
- end
138
- end
203
+ def headers( tag, time, data )
204
+ h = {}
139
205
 
206
+ log.debug "Processing Headers: #{@headers}"
207
+ # A little messy this...
208
+ # Trying to allow for header overrides where a header defined
209
+ # earlier will be used if a later header is returning nil (ie not found and no default)
210
+ h = Hash[ @headers
211
+ .collect{|v| [v.name, v.getValue(data) ]}
212
+ .delete_if{|x| x.last.nil?}
213
+ ]
140
214
 
141
- private
142
- def check_tls_configuration()
143
- if @tls
144
- unless @tls_key && @tls_cert
145
- raise Fluent::ConfigError, "'tls_key' and 'tls_cert' must be all specified if tls is enabled."
146
- end
147
- end
215
+ h[@tag_header] = tag if @tag_header
216
+ h[@time_header] = Time.at(time).utc.to_s if @time_header
217
+
218
+ h
148
219
  end
149
220
 
150
221
  def get_connection_options()
@@ -156,11 +227,11 @@ module Fluent::Plugin
156
227
  tls: @tls || nil,
157
228
  tls_cert: @tls_cert,
158
229
  tls_key: @tls_key,
159
- verify_peer: @tls_verify_peer
230
+ verify_peer: @tls_verify_peer,
231
+ auth_mechanism: @auth_mechanism
160
232
  }
161
233
  opts[:tls_ca_certificates] = @tls_ca_certificates if @tls_ca_certificates
162
234
  return opts
163
235
  end
164
-
165
236
  end
166
237
  end
metadata CHANGED
@@ -1,17 +1,17 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-amqp
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.10.0
4
+ version: 0.14.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hiromi Ishii
8
8
  - Team Giraffi
9
9
  - HiganWorks LLC
10
10
  - Toby Jackson
11
- autorequire:
11
+ autorequire:
12
12
  bindir: bin
13
13
  cert_chain: []
14
- date: 2016-11-17 00:00:00.000000000 Z
14
+ date: 2020-07-12 00:00:00.000000000 Z
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
17
  name: fluentd
@@ -151,7 +151,7 @@ homepage: http://github.com/giraffi/fluent-plugin-amqp
151
151
  licenses:
152
152
  - Apache License, Version 2.0
153
153
  metadata: {}
154
- post_install_message:
154
+ post_install_message:
155
155
  rdoc_options: []
156
156
  require_paths:
157
157
  - lib
@@ -159,16 +159,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
159
159
  requirements:
160
160
  - - ">="
161
161
  - !ruby/object:Gem::Version
162
- version: '0'
162
+ version: 2.1.0
163
163
  required_rubygems_version: !ruby/object:Gem::Requirement
164
164
  requirements:
165
165
  - - ">="
166
166
  - !ruby/object:Gem::Version
167
- version: 2.1.0
167
+ version: '0'
168
168
  requirements: []
169
- rubyforge_project:
170
- rubygems_version: 2.5.1
171
- signing_key:
169
+ rubygems_version: 3.1.2
170
+ signing_key:
172
171
  specification_version: 4
173
172
  summary: AMQP input/output plugin or fluentd
174
173
  test_files: []