fluent-plugin-sumologic_output 1.4.1 → 1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/README.md +5 -3
- data/fluent-plugin-sumologic_output.gemspec +1 -1
- data/lib/fluent/plugin/out_sumologic.rb +35 -6
- data/test/plugin/test_out_sumologic.rb +68 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6db47bbad47c3eb7ea578b1e894ddc972336e785c94bb8af47595481b7e934e2
|
4
|
+
data.tar.gz: 3457257200adc1b655ed1820a5041e29c22531e621aa24a6dd620105ac4559f1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cad8b409225fdd0d96402c17a950c0fb67ad61d0f28ac98a252bac23671cee9a3152738c37f916295c116c0396ede9ea874339c8d4278daf81f698060b56f63d
|
7
|
+
data.tar.gz: 619b7cc028a11d1485b158a08e5f34f00440e5410b14b4283d0dda778f80d22944be9f753275a3028c360e15a69baf437b761a09c7e6b29e2f2c784bf3c0d965
|
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,14 @@
|
|
2
2
|
|
3
3
|
All notable changes to this project will be documented in this file. Tracking did not begin until version 1.10.
|
4
4
|
|
5
|
+
<a name="1.4.0"></a>
|
6
|
+
# [1.4.1] (2019-03-13)
|
7
|
+
|
8
|
+
- Add option for sending metrics in Prometheus format [#39](https://github.com/SumoLogic/fluentd-output-sumologic/pull/39)
|
9
|
+
- Use the build-in extract_placeholders method for header expanding [#40](https://github.com/SumoLogic/fluentd-output-sumologic/pull/40)
|
10
|
+
|
11
|
+
__NOTE:__ there is a breaking change in the placeholders: `tag_parts[n]` is replaced by `tag[n]` [#47](https://github.com/SumoLogic/fluentd-output-sumologic/issues/47)
|
12
|
+
|
5
13
|
<a name="1.4.0"></a>
|
6
14
|
# [1.4.0] (2019-01-16)
|
7
15
|
|
data/README.md
CHANGED
@@ -21,10 +21,10 @@ Configuration options for fluent.conf are:
|
|
21
21
|
* `data_type` - The type of data that will be sent to Sumo Logic, either `logs` or `metrics` (Default is `logs `)
|
22
22
|
* `endpoint` - SumoLogic HTTP Collector URL
|
23
23
|
* `verify_ssl` - Verify ssl certificate. (default is `true`)
|
24
|
-
* `source_category
|
25
|
-
* `source_name
|
24
|
+
* `source_category`<sup>*</sup> - Set _sourceCategory metadata field within SumoLogic (default is `nil`)
|
25
|
+
* `source_name`<sup>*</sup> - Set _sourceName metadata field within SumoLogic - overrides source_name_key (default is `nil`)
|
26
26
|
* `source_name_key` - Set as source::path_key's value so that the source_name can be extracted from Fluentd's buffer (default `source_name`)
|
27
|
-
* `source_host
|
27
|
+
* `source_host`<sup>*</sup> - Set _sourceHost metadata field within SumoLogic (default is `nil`)
|
28
28
|
* `log_format` - Format to post logs into Sumo. (default `json`)
|
29
29
|
* text - Logs will appear in SumoLogic in text format (taken from the field specified in `log_key`)
|
30
30
|
* json - Logs will appear in SumoLogic in json format.
|
@@ -37,6 +37,8 @@ Configuration options for fluent.conf are:
|
|
37
37
|
* `metric_data_format` - The format of metrics you will be sending, either `graphite` or `carbon2` or `prometheus` (Default is `graphite `)
|
38
38
|
* `disable_cookies` - Option to disable cookies on the HTTP Client. (Default is `false `)
|
39
39
|
|
40
|
+
__NOTE:__ <sup>*</sup> [Placeholders](https://docs.fluentd.org/v1.0/articles/buffer-section#placeholders) are supported
|
41
|
+
|
40
42
|
### Example Configuration
|
41
43
|
Reading from the JSON formatted log files with `in_tail` and wildcard filenames:
|
42
44
|
```
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = "fluent-plugin-sumologic_output"
|
7
|
-
gem.version = "1.
|
7
|
+
gem.version = "1.5.0"
|
8
8
|
gem.authors = ["Steven Adams", "Frank Reno"]
|
9
9
|
gem.email = ["stevezau@gmail.com", "frank.reno@me.com"]
|
10
10
|
gem.description = %q{Output plugin to SumoLogic HTTP Endpoint}
|
@@ -12,14 +12,14 @@ class SumologicConnection
|
|
12
12
|
create_http_client(verify_ssl, connect_timeout, proxy_uri, disable_cookies)
|
13
13
|
end
|
14
14
|
|
15
|
-
def publish(raw_data, source_host=nil, source_category=nil, source_name=nil, data_type, metric_data_type)
|
16
|
-
response = http.post(@endpoint, raw_data, request_headers(source_host, source_category, source_name, data_type, metric_data_type))
|
15
|
+
def publish(raw_data, source_host=nil, source_category=nil, source_name=nil, data_type, metric_data_type, collected_fields)
|
16
|
+
response = http.post(@endpoint, raw_data, request_headers(source_host, source_category, source_name, data_type, metric_data_type, collected_fields))
|
17
17
|
unless response.ok?
|
18
18
|
raise RuntimeError, "Failed to send data to HTTP Source. #{response.code} - #{response.body}"
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
22
|
-
def request_headers(source_host, source_category, source_name, data_type, metric_data_format)
|
22
|
+
def request_headers(source_host, source_category, source_name, data_type, metric_data_format, collected_fields)
|
23
23
|
headers = {
|
24
24
|
'X-Sumo-Name' => source_name,
|
25
25
|
'X-Sumo-Category' => source_category,
|
@@ -38,6 +38,9 @@ class SumologicConnection
|
|
38
38
|
raise RuntimeError, "Invalid #{metric_data_format}, must be graphite or carbon2 or prometheus"
|
39
39
|
end
|
40
40
|
end
|
41
|
+
unless collected_fields.nil?
|
42
|
+
headers['X-Sumo-Fields'] = collected_fields
|
43
|
+
end
|
41
44
|
return headers
|
42
45
|
end
|
43
46
|
|
@@ -114,8 +117,8 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
|
|
114
117
|
|
115
118
|
if conf['data_type'].nil? || conf['data_type'] == LOGS_DATA_TYPE
|
116
119
|
unless conf['log_format'].nil?
|
117
|
-
unless conf['log_format'] =~ /\A(?:json|text|json_merge)\z/
|
118
|
-
raise Fluent::ConfigError, "Invalid log_format #{conf['log_format']} must be text, json or
|
120
|
+
unless conf['log_format'] =~ /\A(?:json|text|json_merge|fields)\z/
|
121
|
+
raise Fluent::ConfigError, "Invalid log_format #{conf['log_format']} must be text, json, json_merge or fields"
|
119
122
|
end
|
120
123
|
end
|
121
124
|
end
|
@@ -200,9 +203,28 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
|
|
200
203
|
time.to_s.length == 13 ? time : time * 1000
|
201
204
|
end
|
202
205
|
|
206
|
+
def sumo_fields(sumo_metadata)
|
207
|
+
fields = sumo_metadata['fields'] || ""
|
208
|
+
Hash[
|
209
|
+
fields.split(',').map do |pair|
|
210
|
+
k, v = pair.split('=', 2)
|
211
|
+
[k, v]
|
212
|
+
end
|
213
|
+
]
|
214
|
+
end
|
215
|
+
|
216
|
+
def dump_collected_fields(log_fields)
|
217
|
+
if log_fields.nil?
|
218
|
+
log_fields
|
219
|
+
else
|
220
|
+
log_fields.map{|k,v| "#{k}=#{v}"}.join(',')
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
203
224
|
# This method is called every flush interval. Write the buffer chunk
|
204
225
|
def write(chunk)
|
205
226
|
messages_list = {}
|
227
|
+
log_fields = nil
|
206
228
|
|
207
229
|
# Sort messages
|
208
230
|
chunk.msgpack_each do |time, record|
|
@@ -229,6 +251,12 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
|
|
229
251
|
record = { @timestamp_key => sumo_timestamp(time) }.merge(record)
|
230
252
|
end
|
231
253
|
log = dump_log(merge_json(record))
|
254
|
+
when 'fields'
|
255
|
+
log_fields = sumo_fields(sumo_metadata)
|
256
|
+
if @add_timestamp
|
257
|
+
record = { @timestamp_key => sumo_timestamp(time) }.merge(record)
|
258
|
+
end
|
259
|
+
log = dump_log(record)
|
232
260
|
else
|
233
261
|
if @add_timestamp
|
234
262
|
record = { @timestamp_key => sumo_timestamp(time) }.merge(record)
|
@@ -261,7 +289,8 @@ class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
|
|
261
289
|
source_category =source_category,
|
262
290
|
source_name =source_name,
|
263
291
|
data_type =@data_type,
|
264
|
-
metric_data_format =@metric_data_format
|
292
|
+
metric_data_format =@metric_data_format,
|
293
|
+
collected_fields =dump_collected_fields(log_fields)
|
265
294
|
)
|
266
295
|
end
|
267
296
|
|
@@ -37,7 +37,7 @@ class SumologicOutput < Test::Unit::TestCase
|
|
37
37
|
log_format foo
|
38
38
|
}
|
39
39
|
exception = assert_raise(Fluent::ConfigError) {create_driver(config)}
|
40
|
-
assert_equal("Invalid log_format foo must be text, json or
|
40
|
+
assert_equal("Invalid log_format foo must be text, json, json_merge or fields", exception.message)
|
41
41
|
end
|
42
42
|
|
43
43
|
def test_invalid_metrics_data_type
|
@@ -116,6 +116,27 @@ class SumologicOutput < Test::Unit::TestCase
|
|
116
116
|
times:1
|
117
117
|
end
|
118
118
|
|
119
|
+
def test_emit_empty_fields
|
120
|
+
config = %{
|
121
|
+
endpoint https://collectors.sumologic.com/v1/receivers/http/1234
|
122
|
+
log_format fields
|
123
|
+
source_category test
|
124
|
+
source_host test
|
125
|
+
source_name test
|
126
|
+
|
127
|
+
}
|
128
|
+
driver = create_driver(config)
|
129
|
+
time = event_time
|
130
|
+
stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
|
131
|
+
driver.run do
|
132
|
+
driver.feed("output.test", time, {'message' => 'test'})
|
133
|
+
end
|
134
|
+
assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
|
135
|
+
headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'test', 'X-Sumo-Name'=>'test'},
|
136
|
+
body: /\A{"timestamp":\d+.,"message":"test"}\z/,
|
137
|
+
times:1
|
138
|
+
end
|
139
|
+
|
119
140
|
def test_emit_json_double_encoded
|
120
141
|
config = %{
|
121
142
|
endpoint https://endpoint3.collection.us2.sumologic.com/receiver/v1/http/1234
|
@@ -200,6 +221,52 @@ class SumologicOutput < Test::Unit::TestCase
|
|
200
221
|
times:1
|
201
222
|
end
|
202
223
|
|
224
|
+
def test_emit_with_sumo_metadata_with_fields_json_format
|
225
|
+
config = %{
|
226
|
+
endpoint https://collectors.sumologic.com/v1/receivers/http/1234
|
227
|
+
log_format json
|
228
|
+
}
|
229
|
+
driver = create_driver(config)
|
230
|
+
time = event_time
|
231
|
+
stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
|
232
|
+
ENV['HOST'] = "foo"
|
233
|
+
driver.run do
|
234
|
+
driver.feed("output.test", time, {'foo' => 'bar', 'message' => 'test', '_sumo_metadata' => {
|
235
|
+
"host": "#{ENV['HOST']}",
|
236
|
+
"source": "${tag}",
|
237
|
+
"category": "test",
|
238
|
+
"fields": "foo=bar, sumo = logic"
|
239
|
+
}})
|
240
|
+
end
|
241
|
+
assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
|
242
|
+
headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'foo', 'X-Sumo-Name'=>'output.test'},
|
243
|
+
body: /\A{"timestamp":\d+.,"foo":"bar","message":"test"}\z/,
|
244
|
+
times:1
|
245
|
+
end
|
246
|
+
|
247
|
+
def test_emit_with_sumo_metadata_with_fields_fields_format
|
248
|
+
config = %{
|
249
|
+
endpoint https://collectors.sumologic.com/v1/receivers/http/1234
|
250
|
+
log_format fields
|
251
|
+
}
|
252
|
+
driver = create_driver(config)
|
253
|
+
time = event_time
|
254
|
+
stub_request(:post, 'https://collectors.sumologic.com/v1/receivers/http/1234')
|
255
|
+
ENV['HOST'] = "foo"
|
256
|
+
driver.run do
|
257
|
+
driver.feed("output.test", time, {'foo' => 'shark', 'message' => 'test', '_sumo_metadata' => {
|
258
|
+
"host": "#{ENV['HOST']}",
|
259
|
+
"source": "${tag}",
|
260
|
+
"category": "test",
|
261
|
+
"fields": "foo=bar, sumo = logic"
|
262
|
+
}})
|
263
|
+
end
|
264
|
+
assert_requested :post, "https://collectors.sumologic.com/v1/receivers/http/1234",
|
265
|
+
headers: {'X-Sumo-Category'=>'test', 'X-Sumo-Client'=>'fluentd-output', 'X-Sumo-Host'=>'foo', 'X-Sumo-Name'=>'output.test', 'X-Sumo-Fields' => 'foo=bar, sumo = logic'},
|
266
|
+
body: /\A{"timestamp":\d+.,"foo":"shark","message":"test"}\z/,
|
267
|
+
times:1
|
268
|
+
end
|
269
|
+
|
203
270
|
def test_emit_with_sumo_metadata
|
204
271
|
config = %{
|
205
272
|
endpoint https://collectors.sumologic.com/v1/receivers/http/1234
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sumologic_output
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Steven Adams
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-
|
12
|
+
date: 2019-06-26 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -134,7 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
134
134
|
- !ruby/object:Gem::Version
|
135
135
|
version: '0'
|
136
136
|
requirements: []
|
137
|
-
rubygems_version: 3.0.
|
137
|
+
rubygems_version: 3.0.4
|
138
138
|
signing_key:
|
139
139
|
specification_version: 4
|
140
140
|
summary: Output plugin to SumoLogic HTTP Endpoint
|