fluent-plugin-splunk-hec 1.1.2 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +2 -0
- data/Gemfile.lock +125 -14
- data/README.md +136 -74
- data/Rakefile +6 -1
- data/VERSION +1 -1
- data/fluent-plugin-splunk-hec.gemspec +9 -3
- data/lib/fluent/plugin/out_splunk.rb +313 -0
- data/lib/fluent/plugin/{out_splunk_hec → out_splunk}/match_formatter.rb +5 -3
- data/lib/fluent/plugin/out_splunk/version.rb +3 -0
- data/lib/fluent/plugin/out_splunk_hec.rb +130 -190
- data/lib/fluent/plugin/out_splunk_hec/version.rb +2 -0
- data/lib/fluent/plugin/out_splunk_ingest_api.rb +109 -0
- data/test/fluent/plugin/out_splunk_hec_test.rb +232 -221
- data/test/fluent/plugin/out_splunk_ingest_api_test.rb +244 -0
- data/test/test_helper.rb +10 -7
- metadata +82 -23
- data/test/lib/webmock/http_lib_adapters/httpclient_adapter.rb +0 -0
data/Rakefile
CHANGED
@@ -1,5 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'bundler/gem_tasks'
|
2
4
|
require 'rake/testtask'
|
5
|
+
require 'rubocop/rake_task'
|
6
|
+
|
7
|
+
RuboCop::RakeTask.new
|
3
8
|
|
4
9
|
Rake::TestTask.new(:test) do |t|
|
5
10
|
t.libs << 'test'
|
@@ -9,4 +14,4 @@ Rake::TestTask.new(:test) do |t|
|
|
9
14
|
t.warning = false
|
10
15
|
end
|
11
16
|
|
12
|
-
task :
|
17
|
+
task default: :test
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.2.0
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
Gem::Specification.new do |spec|
|
2
4
|
spec.name = 'fluent-plugin-splunk-hec'
|
3
5
|
spec.version = File.read('VERSION')
|
@@ -31,15 +33,19 @@ Gem::Specification.new do |spec|
|
|
31
33
|
|
32
34
|
spec.required_ruby_version = '>= 2.3.0'
|
33
35
|
|
34
|
-
spec.add_runtime_dependency '
|
36
|
+
spec.add_runtime_dependency 'fluent-plugin-kubernetes_metadata_filter', '= 2.1.2'
|
37
|
+
spec.add_runtime_dependency 'fluentd', '= 1.4'
|
35
38
|
spec.add_runtime_dependency 'multi_json', '~> 1.13'
|
36
39
|
spec.add_runtime_dependency 'net-http-persistent', '~> 3.0'
|
40
|
+
spec.add_runtime_dependency 'openid_connect', '~> 1.1.6'
|
41
|
+
spec.add_runtime_dependency 'prometheus-client', '~> 0.9.0'
|
37
42
|
|
38
43
|
spec.add_development_dependency 'bundler', '~> 2.0'
|
39
44
|
spec.add_development_dependency 'rake', '~> 12.0'
|
40
45
|
# required by fluent/test.rb
|
41
|
-
spec.add_development_dependency 'test-unit', '~> 3.0'
|
42
46
|
spec.add_development_dependency 'minitest', '~> 5.0'
|
43
|
-
spec.add_development_dependency '
|
47
|
+
spec.add_development_dependency 'rubocop', '~> 0.63.1'
|
44
48
|
spec.add_development_dependency 'simplecov', '~> 0.16.1'
|
49
|
+
spec.add_development_dependency 'test-unit', '~> 3.0'
|
50
|
+
spec.add_development_dependency 'webmock', '~> 3.5.0'
|
45
51
|
end
|
@@ -0,0 +1,313 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'fluent/output'
|
4
|
+
require 'fluent/plugin/output'
|
5
|
+
require 'fluent/plugin/formatter'
|
6
|
+
require 'prometheus/client'
|
7
|
+
require 'benchmark'
|
8
|
+
|
9
|
+
module Fluent::Plugin
|
10
|
+
class SplunkOutput < Fluent::BufferedOutput
|
11
|
+
helpers :formatter
|
12
|
+
|
13
|
+
autoload :VERSION, 'fluent/plugin/out_splunk/version'
|
14
|
+
autoload :MatchFormatter, 'fluent/plugin/out_splunk/match_formatter'
|
15
|
+
|
16
|
+
KEY_FIELDS = %w[index host source sourcetype metric_name metric_value].freeze
|
17
|
+
TAG_PLACEHOLDER = '${tag}'
|
18
|
+
|
19
|
+
desc 'The host field for events, by default it uses the hostname of the machine that runnning fluentd. This is exclusive with `host_key`.'
|
20
|
+
config_param :host, :string, default: nil
|
21
|
+
|
22
|
+
desc 'Field name to contain host. This is exclusive with `host`.'
|
23
|
+
config_param :host_key, :string, default: nil
|
24
|
+
|
25
|
+
desc 'The source field for events, when not set, will be decided by HEC. This is exclusive with `source_key`.'
|
26
|
+
config_param :source, :string, default: nil
|
27
|
+
|
28
|
+
desc 'Field name to contain source. This is exclusive with `source`.'
|
29
|
+
config_param :source_key, :string, default: nil
|
30
|
+
|
31
|
+
desc 'The sourcetype field for events, when not set, will be decided by HEC. This is exclusive with `sourcetype_key`.'
|
32
|
+
config_param :sourcetype, :string, default: nil
|
33
|
+
|
34
|
+
desc 'Field name to contain sourcetype. This is exclusive with `sourcetype`.'
|
35
|
+
config_param :sourcetype_key, :string, default: nil
|
36
|
+
|
37
|
+
desc 'Field name to contain Splunk event time. By default will use fluentd\'d time'
|
38
|
+
config_param :time_key, :string, default: nil
|
39
|
+
|
40
|
+
desc 'The Splunk index to index events. When not set, will be decided by HEC. This is exclusive with `index_key`'
|
41
|
+
config_param :index, :string, default: nil
|
42
|
+
|
43
|
+
desc 'Field name to contain Splunk index name. This is exclusive with `index`.'
|
44
|
+
config_param :index_key, :string, default: nil
|
45
|
+
|
46
|
+
desc 'When set to true, all fields defined in `index_key`, `host_key`, `source_key`, `sourcetype_key`, `metric_name_key`, `metric_value_key` will not be removed from the original event.'
|
47
|
+
config_param :keep_keys, :bool, default: false
|
48
|
+
|
49
|
+
desc 'Define index-time fields for event data type, or metric dimensions for metric data type. Null value fields will be removed.'
|
50
|
+
config_section :fields, init: false, multi: false, required: false do
|
51
|
+
# this is blank on purpose
|
52
|
+
end
|
53
|
+
|
54
|
+
config_section :format do
|
55
|
+
config_set_default :usage, '**'
|
56
|
+
config_set_default :@type, 'json'
|
57
|
+
config_set_default :add_newline, false
|
58
|
+
end
|
59
|
+
|
60
|
+
desc <<~DESC
|
61
|
+
Whether to allow non-UTF-8 characters in user logs. If set to true, any
|
62
|
+
non-UTF-8 character would be replaced by the string specified by
|
63
|
+
`non_utf8_replacement_string`. If set to false, any non-UTF-8 character
|
64
|
+
would trigger the plugin to error out.
|
65
|
+
DESC
|
66
|
+
config_param :coerce_to_utf8, :bool, default: true
|
67
|
+
|
68
|
+
desc <<~DESC
|
69
|
+
If `coerce_to_utf8` is set to true, any non-UTF-8 character would be
|
70
|
+
replaced by the string specified here.
|
71
|
+
DESC
|
72
|
+
config_param :non_utf8_replacement_string, :string, default: ' '
|
73
|
+
|
74
|
+
def initialize
|
75
|
+
super
|
76
|
+
@registry = ::Prometheus::Client.registry
|
77
|
+
end
|
78
|
+
|
79
|
+
def configure(conf)
|
80
|
+
super
|
81
|
+
check_conflict
|
82
|
+
@api = construct_api
|
83
|
+
prepare_key_fields
|
84
|
+
configure_fields(conf)
|
85
|
+
configure_metrics(conf)
|
86
|
+
|
87
|
+
# @formatter_configs is from formatter helper
|
88
|
+
@formatters = @formatter_configs.map do |section|
|
89
|
+
MatchFormatter.new section.usage, formatter_create(usage: section.usage)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def write(chunk)
|
94
|
+
log.trace { "#{self.class}: Received new chunk, size=#{chunk.read.bytesize}" }
|
95
|
+
|
96
|
+
t = Benchmark.realtime do
|
97
|
+
write_to_splunk(chunk)
|
98
|
+
end
|
99
|
+
|
100
|
+
@metrics[:record_counter].increment(metric_labels, chunk.size_of_events)
|
101
|
+
@metrics[:bytes_counter].increment(metric_labels, chunk.bytesize)
|
102
|
+
@metrics[:write_records_histogram].observe(metric_labels, chunk.size_of_events)
|
103
|
+
@metrics[:write_bytes_histogram].observe(metric_labels, chunk.bytesize)
|
104
|
+
@metrics[:write_latency_histogram].observe(metric_labels, t)
|
105
|
+
end
|
106
|
+
|
107
|
+
def write_to_splunk(_chunk)
|
108
|
+
raise NotImplementedError("Child class should implement 'write_to_splunk'")
|
109
|
+
end
|
110
|
+
|
111
|
+
def construct_api
|
112
|
+
raise NotImplementedError("Child class should implement 'construct_api'")
|
113
|
+
end
|
114
|
+
|
115
|
+
protected
|
116
|
+
|
117
|
+
def prepare_event_payload(tag, time, record)
|
118
|
+
{
|
119
|
+
host: @host ? @host.call(tag, record) : @default_host,
|
120
|
+
# From the API reference
|
121
|
+
# http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
|
122
|
+
# `time` should be a string or unsigned integer.
|
123
|
+
# That's why we use `to_s` here.
|
124
|
+
time: time.to_f.to_s
|
125
|
+
}.tap do |payload|
|
126
|
+
payload[:index] = @index.call(tag, record) if @index
|
127
|
+
payload[:source] = @source.call(tag, record) if @source
|
128
|
+
payload[:sourcetype] = @sourcetype.call(tag, record) if @sourcetype
|
129
|
+
|
130
|
+
# delete nil fields otherwise will get format error from HEC
|
131
|
+
%i[host index source sourcetype].each { |f| payload.delete f if payload[f].nil? }
|
132
|
+
|
133
|
+
if @extra_fields
|
134
|
+
payload[:fields] = @extra_fields.map { |name, field| [name, record[field]] }.to_h
|
135
|
+
payload[:fields].compact!
|
136
|
+
# if a field is already in indexed fields, then remove it from the original event
|
137
|
+
@extra_fields.values.each { |field| record.delete field }
|
138
|
+
end
|
139
|
+
if formatter = @formatters.find { |f| f.match? tag }
|
140
|
+
record = formatter.format(tag, time, record)
|
141
|
+
end
|
142
|
+
payload[:event] = convert_to_utf8 record
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
def format_event(tag, time, record)
|
147
|
+
MultiJson.dump(prepare_event_payload(tag, time, record))
|
148
|
+
end
|
149
|
+
|
150
|
+
def process_response(response, _request_body)
|
151
|
+
log.trace { "[Response] POST #{@api}: #{response.inspect}" }
|
152
|
+
|
153
|
+
@metrics[:status_counter].increment(metric_labels(status: response.code.to_s))
|
154
|
+
|
155
|
+
# raise Exception to utilize Fluentd output plugin retry mechanism
|
156
|
+
raise "Server error (#{response.code}) for POST #{@api}, response: #{response.body}" if response.code.to_s.start_with?('5')
|
157
|
+
|
158
|
+
# For both success response (2xx) and client errors (4xx), we will consume the chunk.
|
159
|
+
# Because there probably a bug in the code if we get 4xx errors, retry won't do any good.
|
160
|
+
unless response.code.to_s.start_with?('2')
|
161
|
+
log.error "#{self.class}: Failed POST to #{@api}, response: #{response.body}"
|
162
|
+
log.error { "#{self.class}: Failed request body: #{post.body}" }
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
private
|
167
|
+
|
168
|
+
def check_conflict
|
169
|
+
KEY_FIELDS.each do |f|
|
170
|
+
kf = "#{f}_key"
|
171
|
+
raise Fluent::ConfigError, "Can not set #{f} and #{kf} at the same time." \
|
172
|
+
if %W[@#{f} @#{kf}].all? &method(:instance_variable_get)
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
def prepare_key_fields
|
177
|
+
KEY_FIELDS.each do |f|
|
178
|
+
v = instance_variable_get "@#{f}_key"
|
179
|
+
if v
|
180
|
+
attrs = v.split('.').freeze
|
181
|
+
if @keep_keys
|
182
|
+
instance_variable_set "@#{f}", ->(_, record) { attrs.inject(record) { |o, k| o[k] } }
|
183
|
+
else
|
184
|
+
instance_variable_set "@#{f}", lambda { |_, record|
|
185
|
+
attrs[0...-1].inject(record) { |o, k| o[k] }.delete(attrs[-1])
|
186
|
+
}
|
187
|
+
end
|
188
|
+
else
|
189
|
+
v = instance_variable_get "@#{f}"
|
190
|
+
next unless v
|
191
|
+
|
192
|
+
if v == TAG_PLACEHOLDER
|
193
|
+
instance_variable_set "@#{f}", ->(tag, _) { tag }
|
194
|
+
else
|
195
|
+
instance_variable_set "@#{f}", ->(_, _) { v }
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
# <fields> directive, which defines:
|
202
|
+
# * when data_type is event, index-time fields
|
203
|
+
# * when data_type is metric, metric dimensions
|
204
|
+
def configure_fields(conf)
|
205
|
+
# This loop looks dump, but it is used to suppress the unused parameter configuration warning
|
206
|
+
# Learned from `filter_record_transformer`.
|
207
|
+
conf.elements.select { |element| element.name == 'fields' }.each do |element|
|
208
|
+
element.each_pair { |k, _v| element.key?(k) }
|
209
|
+
end
|
210
|
+
|
211
|
+
return unless @fields
|
212
|
+
|
213
|
+
@extra_fields = @fields.corresponding_config_element.map do |k, v|
|
214
|
+
[k, v.empty? ? k : v]
|
215
|
+
end.to_h
|
216
|
+
end
|
217
|
+
|
218
|
+
def pick_custom_format_method
|
219
|
+
if @data_type == :event
|
220
|
+
define_singleton_method :format, method(:format_event)
|
221
|
+
else
|
222
|
+
define_singleton_method :format, method(:format_metric)
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
226
|
+
def configure_metrics(conf)
|
227
|
+
@metric_labels = {
|
228
|
+
type: conf['@type'],
|
229
|
+
plugin_id: plugin_id
|
230
|
+
}
|
231
|
+
|
232
|
+
@metrics = {
|
233
|
+
record_counter: register_metric(::Prometheus::Client::Counter.new(
|
234
|
+
:splunk_output_write_records_count,
|
235
|
+
'The number of log records being sent'
|
236
|
+
)),
|
237
|
+
bytes_counter: register_metric(::Prometheus::Client::Counter.new(
|
238
|
+
:splunk_output_write_bytes_count,
|
239
|
+
'The number of log bytes being sent'
|
240
|
+
)),
|
241
|
+
status_counter: register_metric(::Prometheus::Client::Counter.new(
|
242
|
+
:splunk_output_write_status_count,
|
243
|
+
'The count of sends by response_code'
|
244
|
+
)),
|
245
|
+
write_bytes_histogram: register_metric(::Prometheus::Client::Histogram.new(
|
246
|
+
:splunk_output_write_payload_bytes,
|
247
|
+
'The size of the write payload in bytes', {}, [1024, 23_937, 47_875, 95_750, 191_500, 383_000, 766_000, 1_149_000]
|
248
|
+
)),
|
249
|
+
write_records_histogram: register_metric(::Prometheus::Client::Histogram.new(
|
250
|
+
:splunk_output_write_payload_records,
|
251
|
+
'The number of records written per write', {}, [1, 10, 25, 100, 200, 300, 500, 750, 1000, 1500]
|
252
|
+
)),
|
253
|
+
write_latency_histogram: register_metric(::Prometheus::Client::Histogram.new(
|
254
|
+
:splunk_output_write_latency_seconds,
|
255
|
+
'The latency of writes'
|
256
|
+
))
|
257
|
+
}
|
258
|
+
end
|
259
|
+
|
260
|
+
# Tag metrics with the type string that was used to register the plugin
|
261
|
+
def metric_labels(other_labels = {})
|
262
|
+
@metric_labels.merge other_labels
|
263
|
+
end
|
264
|
+
|
265
|
+
# Encode as UTF-8. If 'coerce_to_utf8' is set to true in the config, any
|
266
|
+
# non-UTF-8 character would be replaced by the string specified by
|
267
|
+
# 'non_utf8_replacement_string'. If 'coerce_to_utf8' is set to false, any
|
268
|
+
# non-UTF-8 character would trigger the plugin to error out.
|
269
|
+
# Thanks to
|
270
|
+
# https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud/blob/dbc28575/lib/fluent/plugin/out_google_cloud.rb#L1284
|
271
|
+
def convert_to_utf8(input)
|
272
|
+
if input.is_a?(Hash)
|
273
|
+
record = {}
|
274
|
+
input.each do |key, value|
|
275
|
+
record[convert_to_utf8(key)] = convert_to_utf8(value)
|
276
|
+
end
|
277
|
+
|
278
|
+
return record
|
279
|
+
end
|
280
|
+
return input.map { |value| convert_to_utf8(value) } if input.is_a?(Array)
|
281
|
+
return input unless input.respond_to?(:encode)
|
282
|
+
|
283
|
+
if @coerce_to_utf8
|
284
|
+
input.encode(
|
285
|
+
'utf-8',
|
286
|
+
invalid: :replace,
|
287
|
+
undef: :replace,
|
288
|
+
replace: @non_utf8_replacement_string
|
289
|
+
)
|
290
|
+
else
|
291
|
+
begin
|
292
|
+
input.encode('utf-8')
|
293
|
+
rescue EncodingError
|
294
|
+
log.error do
|
295
|
+
'Encountered encoding issues potentially due to non ' \
|
296
|
+
'UTF-8 characters. To allow non-UTF-8 characters and ' \
|
297
|
+
'replace them with spaces, please set "coerce_to_utf8" ' \
|
298
|
+
'to true.'
|
299
|
+
end
|
300
|
+
raise
|
301
|
+
end
|
302
|
+
end
|
303
|
+
end
|
304
|
+
|
305
|
+
def register_metric(metric)
|
306
|
+
if !@registry.exist?(metric.name)
|
307
|
+
@registry.register(metric)
|
308
|
+
else
|
309
|
+
@registry.get(metric.name)
|
310
|
+
end
|
311
|
+
end
|
312
|
+
end
|
313
|
+
end
|
@@ -1,11 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'fluent/match'
|
2
4
|
|
3
|
-
class Fluent::Plugin::
|
5
|
+
class Fluent::Plugin::SplunkOutput::MatchFormatter
|
4
6
|
def initialize(pattern, formatter)
|
5
7
|
# based on fluentd/lib/fluent/event_router.rb
|
6
|
-
patterns = pattern.split(/\s+/).map
|
8
|
+
patterns = pattern.split(/\s+/).map do |str|
|
7
9
|
Fluent::MatchPattern.create(str)
|
8
|
-
|
10
|
+
end
|
9
11
|
@pattern =
|
10
12
|
if patterns.length == 1
|
11
13
|
patterns[0]
|
@@ -1,14 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require 'fluent/output'
|
4
|
+
require 'fluent/plugin/output'
|
5
|
+
require 'fluent/plugin/formatter'
|
6
|
+
require 'fluent/plugin/out_splunk'
|
5
7
|
|
6
8
|
require 'openssl'
|
7
9
|
require 'multi_json'
|
8
10
|
require 'net/http/persistent'
|
9
11
|
|
10
12
|
module Fluent::Plugin
|
11
|
-
class SplunkHecOutput <
|
13
|
+
class SplunkHecOutput < SplunkOutput
|
12
14
|
Fluent::Plugin.register_output('splunk_hec', self)
|
13
15
|
|
14
16
|
helpers :formatter
|
@@ -19,10 +21,10 @@ module Fluent::Plugin
|
|
19
21
|
KEY_FIELDS = %w[index time host source sourcetype metric_name metric_value].freeze
|
20
22
|
TAG_PLACEHOLDER = '${tag}'.freeze
|
21
23
|
|
22
|
-
MISSING_FIELD = Hash.new
|
24
|
+
MISSING_FIELD = Hash.new do |_h, k|
|
23
25
|
$log.warn "expected field #{k} but it's missing" if defined?($log)
|
24
26
|
MISSING_FIELD
|
25
|
-
|
27
|
+
end.freeze
|
26
28
|
|
27
29
|
desc 'Protocol to use to call HEC API.'
|
28
30
|
config_param :protocol, :enum, list: %i[http https], default: :https
|
@@ -75,31 +77,13 @@ module Fluent::Plugin
|
|
75
77
|
desc 'Field name to contain Splunk index name. This is exclusive with `index`.'
|
76
78
|
config_param :index_key, :string, default: nil
|
77
79
|
|
78
|
-
desc "The host field for events, by default it uses the hostname of the machine that runnning fluentd. This is exclusive with `host_key`."
|
79
|
-
config_param :host, :string, default: nil
|
80
|
-
|
81
|
-
desc 'Field name to contain host. This is exclusive with `host`.'
|
82
|
-
config_param :host_key, :string, default: nil
|
83
|
-
|
84
|
-
desc 'The source field for events, when not set, will be decided by HEC. This is exclusive with `source_key`.'
|
85
|
-
config_param :source, :string, default: nil
|
86
|
-
|
87
|
-
desc 'Field name to contain source. This is exclusive with `source`.'
|
88
|
-
config_param :source_key, :string, default: nil
|
89
|
-
|
90
|
-
desc 'The sourcetype field for events, when not set, will be decided by HEC. This is exclusive with `sourcetype_key`.'
|
91
|
-
config_param :sourcetype, :string, default: nil
|
92
|
-
|
93
|
-
desc 'Field name to contain sourcetype. This is exclusive with `sourcetype`.'
|
94
|
-
config_param :sourcetype_key, :string, default: nil
|
95
|
-
|
96
80
|
desc 'When `data_type` is set to "metric", by default it will treat every key-value pair in the income event as a metric name-metric value pair. Set `metrics_from_event` to `false` to disable this behavior and use `metric_name_key` and `metric_value_key` to define metrics.'
|
97
81
|
config_param :metrics_from_event, :bool, default: true
|
98
82
|
|
99
|
-
desc
|
83
|
+
desc 'Field name to contain metric name. This is exclusive with `metrics_from_event`, when this is set, `metrics_from_event` will be set to `false`.'
|
100
84
|
config_param :metric_name_key, :string, default: nil
|
101
85
|
|
102
|
-
desc
|
86
|
+
desc 'Field name to contain metric value, this is required when `metric_name_key` is set.'
|
103
87
|
config_param :metric_value_key, :string, default: nil
|
104
88
|
|
105
89
|
desc 'When set to true, all fields defined in `index_key`, `host_key`, `source_key`, `sourcetype_key`, `metric_name_key`, `metric_value_key` will not be removed from the original event.'
|
@@ -109,7 +93,7 @@ module Fluent::Plugin
|
|
109
93
|
config_section :fields, init: false, multi: false, required: false do
|
110
94
|
# this is blank on purpose
|
111
95
|
end
|
112
|
-
|
96
|
+
|
113
97
|
config_section :format do
|
114
98
|
config_set_default :usage, '**'
|
115
99
|
config_set_default :@type, 'json'
|
@@ -139,47 +123,42 @@ module Fluent::Plugin
|
|
139
123
|
def configure(conf)
|
140
124
|
super
|
141
125
|
|
142
|
-
check_conflict
|
143
126
|
check_metric_configs
|
144
|
-
construct_api
|
145
|
-
prepare_key_fields
|
146
|
-
configure_fields(conf)
|
147
127
|
pick_custom_format_method
|
148
|
-
|
149
|
-
# @formatter_configs is from formatter helper
|
150
|
-
@formatters = @formatter_configs.map { |section|
|
151
|
-
MatchFormatter.new section.usage, formatter_create(usage: section.usage)
|
152
|
-
}
|
153
128
|
end
|
154
129
|
|
155
130
|
def start
|
156
131
|
super
|
132
|
+
@conn = Net::HTTP::Persistent.new.tap do |c|
|
133
|
+
c.verify_mode = @insecure_ssl ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
|
134
|
+
c.cert = OpenSSL::X509::Certificate.new File.read(@client_cert) if @client_cert
|
135
|
+
c.key = OpenSSL::PKey::RSA.new File.read(@client_key) if @client_key
|
136
|
+
c.ca_file = @ca_file
|
137
|
+
c.ca_path = @ca_path
|
138
|
+
c.ciphers = @ssl_ciphers
|
157
139
|
|
158
|
-
|
140
|
+
c.override_headers['Content-Type'] = 'application/json'
|
141
|
+
c.override_headers['User-Agent'] = "fluent-plugin-splunk_hec_out/#{VERSION}"
|
142
|
+
c.override_headers['Authorization'] = "Splunk #{@hec_token}"
|
143
|
+
end
|
159
144
|
end
|
160
145
|
|
161
|
-
def
|
162
|
-
|
146
|
+
def shutdown
|
147
|
+
super
|
148
|
+
@conn.shutdown
|
163
149
|
end
|
164
150
|
|
165
|
-
def
|
166
|
-
|
167
|
-
send_to_hec chunk
|
151
|
+
def format(tag, time, record)
|
152
|
+
# this method will be replaced in `configure`
|
168
153
|
end
|
169
154
|
|
170
155
|
def multi_workers_ready?
|
171
156
|
true
|
172
157
|
end
|
173
158
|
|
174
|
-
|
159
|
+
protected
|
175
160
|
|
176
|
-
|
177
|
-
KEY_FIELDS.each { |f|
|
178
|
-
kf = "#{f}_key"
|
179
|
-
raise Fluent::ConfigError, "Can not set #{f} and #{kf} at the same time." \
|
180
|
-
if %W[@#{f} @#{kf}].all? &method(:instance_variable_get)
|
181
|
-
}
|
182
|
-
end
|
161
|
+
private
|
183
162
|
|
184
163
|
def check_metric_configs
|
185
164
|
return unless @data_type == :metric
|
@@ -188,71 +167,57 @@ module Fluent::Plugin
|
|
188
167
|
|
189
168
|
return if @metrics_from_event
|
190
169
|
|
191
|
-
raise Fluent::ConfigError,
|
192
|
-
|
193
|
-
raise Fluent::ConfigError, "`metric_value_key` is required when `metric_name_key` is set." unless @metric_value_key
|
170
|
+
raise Fluent::ConfigError, '`metric_name_key` is required when `metrics_from_event` is `false`.' unless @metric_name_key
|
171
|
+
raise Fluent::ConfigError, '`metric_value_key` is required when `metric_name_key` is set.' unless @metric_value_key
|
194
172
|
end
|
195
173
|
|
196
|
-
def
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
if v == TAG_PLACEHOLDER
|
213
|
-
instance_variable_set "@#{f}", ->(tag, _) { tag }
|
214
|
-
else
|
215
|
-
instance_variable_set "@#{f}", ->(_, _) { v }
|
216
|
-
end
|
217
|
-
end
|
218
|
-
}
|
219
|
-
end
|
220
|
-
|
221
|
-
# <fields> directive, which defines:
|
222
|
-
# * when data_type is event, index-time fields
|
223
|
-
# * when data_type is metric, metric dimensions
|
224
|
-
def configure_fields(conf)
|
225
|
-
# This loop looks dump, but it is used to suppress the unused parameter configuration warning
|
226
|
-
# Learned from `filter_record_transformer`.
|
227
|
-
conf.elements.select { |element| element.name == 'fields' }.each do |element|
|
228
|
-
element.each_pair { |k, v| element.has_key?(k) }
|
229
|
-
end
|
174
|
+
def format_event(tag, time, record)
|
175
|
+
MultiJson.dump({
|
176
|
+
host: @host ? @host.(tag, record) : @default_host,
|
177
|
+
# From the API reference
|
178
|
+
# http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
|
179
|
+
# `time` should be a string or unsigned integer.
|
180
|
+
# That's why we use the to_string function here.
|
181
|
+
time: time.to_f.to_s
|
182
|
+
}.tap { |payload|
|
183
|
+
if @time
|
184
|
+
time_value = @time.(tag, record)
|
185
|
+
# if no value is found don't override and use fluentd's time
|
186
|
+
if !time_value.nil?
|
187
|
+
payload[:time] = time_value
|
188
|
+
end
|
189
|
+
end
|
230
190
|
|
231
|
-
|
191
|
+
payload[:index] = @index.(tag, record) if @index
|
192
|
+
payload[:source] = @source.(tag, record) if @source
|
193
|
+
payload[:sourcetype] = @sourcetype.(tag, record) if @sourcetype
|
232
194
|
|
233
|
-
|
234
|
-
|
235
|
-
}.to_h
|
236
|
-
end
|
195
|
+
# delete nil fields otherwise will get formet error from HEC
|
196
|
+
%i[host index source sourcetype].each { |f| payload.delete f if payload[f].nil? }
|
237
197
|
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
198
|
+
if @extra_fields
|
199
|
+
payload[:fields] = @extra_fields.map { |name, field| [name, record[field]] }.to_h
|
200
|
+
payload[:fields].delete_if { |_k,v| v.nil? }
|
201
|
+
# if a field is already in indexed fields, then remove it from the original event
|
202
|
+
@extra_fields.values.each { |field| record.delete field }
|
203
|
+
end
|
204
|
+
if formatter = @formatters.find { |f| f.match? tag }
|
205
|
+
record = formatter.format(tag, time, record)
|
206
|
+
end
|
207
|
+
payload[:event] = convert_to_utf8 record
|
208
|
+
})
|
244
209
|
end
|
245
210
|
|
246
|
-
def
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
211
|
+
def format_metric(tag, time, record)
|
212
|
+
payload = {
|
213
|
+
host: @host ? @host.call(tag, record) : @default_host,
|
214
|
+
# From the API reference
|
215
|
+
# http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
|
216
|
+
# `time` should be a string or unsigned integer.
|
217
|
+
# That's why we use `to_s` here.
|
218
|
+
time: time.to_f.to_s,
|
219
|
+
event: 'metric'
|
220
|
+
}.tap do |payload|
|
256
221
|
if @time
|
257
222
|
time_value = @time.(tag, record)
|
258
223
|
# if no value is found don't override and use fluentd's time
|
@@ -260,71 +225,42 @@ module Fluent::Plugin
|
|
260
225
|
payload[:time] = time_value
|
261
226
|
end
|
262
227
|
end
|
228
|
+
end
|
229
|
+
payload[:index] = @index.call(tag, record) if @index
|
230
|
+
payload[:source] = @source.call(tag, record) if @source
|
231
|
+
payload[:sourcetype] = @sourcetype.call(tag, record) if @sourcetype
|
232
|
+
|
233
|
+
unless @metrics_from_event
|
234
|
+
fields = {
|
235
|
+
metric_name: @metric_name.call(tag, record),
|
236
|
+
_value: @metric_value.call(tag, record)
|
237
|
+
}
|
238
|
+
|
239
|
+
if @extra_fields
|
240
|
+
fields.update @extra_fields.map { |name, field| [name, record[field]] }.to_h
|
241
|
+
fields.delete_if { |_k,v| v.nil? }
|
242
|
+
else
|
243
|
+
fields.update record
|
244
|
+
end
|
263
245
|
|
264
|
-
|
265
|
-
payload[:source] = @source.(tag, record) if @source
|
266
|
-
payload[:sourcetype] = @sourcetype.(tag, record) if @sourcetype
|
267
|
-
|
268
|
-
# delete nil fields otherwise will get formet error from HEC
|
269
|
-
%i[host index source sourcetype].each { |f| payload.delete f if payload[f].nil? }
|
270
|
-
|
271
|
-
if @extra_fields
|
272
|
-
payload[:fields] = @extra_fields.map { |name, field| [name, record[field]] }.to_h
|
273
|
-
payload[:fields].delete_if { |_k,v| v.nil? }
|
274
|
-
# if a field is already in indexed fields, then remove it from the original event
|
275
|
-
@extra_fields.values.each { |field| record.delete field }
|
276
|
-
end
|
277
|
-
if formatter = @formatters.find { |f| f.match? tag }
|
278
|
-
record = formatter.format(tag, time, record)
|
279
|
-
end
|
280
|
-
payload[:event] = convert_to_utf8 record
|
281
|
-
})
|
282
|
-
end
|
246
|
+
fields.delete_if { |_k,v| v.nil? }
|
283
247
|
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
# From the API reference
|
288
|
-
# http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
|
289
|
-
# `time` should be a string or unsigned integer.
|
290
|
-
# That's why we use `to_s` here.
|
291
|
-
time: time.to_f.to_s,
|
292
|
-
event: 'metric'
|
293
|
-
}
|
294
|
-
payload[:index] = @index.(tag, record) if @index
|
295
|
-
payload[:source] = @source.(tag, record) if @source
|
296
|
-
payload[:sourcetype] = @sourcetype.(tag, record) if @sourcetype
|
297
|
-
|
298
|
-
if not @metrics_from_event
|
299
|
-
fields = {
|
300
|
-
metric_name: @metric_name.(tag, record),
|
301
|
-
_value: @metric_value.(tag, record)
|
302
|
-
}
|
303
|
-
|
304
|
-
if @extra_fields
|
305
|
-
fields.update @extra_fields.map { |name, field| [name, record[field]] }.to_h
|
306
|
-
else
|
307
|
-
fields.update record
|
308
|
-
end
|
309
|
-
|
310
|
-
fields.delete_if { |_k,v| v.nil? }
|
311
|
-
|
312
|
-
payload[:fields] = convert_to_utf8 fields
|
313
|
-
|
314
|
-
return MultiJson.dump(payload)
|
248
|
+
payload[:fields] = convert_to_utf8 fields
|
249
|
+
|
250
|
+
return MultiJson.dump(payload)
|
315
251
|
end
|
316
252
|
|
317
253
|
# when metrics_from_event is true, generate one metric event for each key-value in record
|
318
|
-
payloads = record.map
|
319
|
-
|
320
|
-
|
254
|
+
payloads = record.map do |key, value|
|
255
|
+
{ fields: { metric_name: key, _value: value } }.merge! payload
|
256
|
+
end
|
321
257
|
|
322
258
|
payloads.map!(&MultiJson.method(:dump)).join
|
323
259
|
end
|
324
260
|
|
325
261
|
def construct_api
|
326
|
-
|
327
|
-
rescue
|
262
|
+
URI("#{@protocol}://#{@hec_host}:#{@hec_port}/services/collector")
|
263
|
+
rescue StandardError
|
328
264
|
raise Fluent::ConfigError, "hec_host (#{@hec_host}) and/or hec_port (#{@hec_port}) are invalid."
|
329
265
|
end
|
330
266
|
|
@@ -346,14 +282,15 @@ module Fluent::Plugin
|
|
346
282
|
end
|
347
283
|
end
|
348
284
|
|
349
|
-
def
|
350
|
-
post = Net::HTTP::Post.new @
|
285
|
+
def write_to_splunk(chunk)
|
286
|
+
post = Net::HTTP::Post.new @api.request_uri
|
351
287
|
post.body = chunk.read
|
352
|
-
log.debug { "Sending #{post.body.bytesize}
|
288
|
+
log.debug { "[Sending] Chunk: #{dump_unique_id_hex(chunk.unique_id)}(#{post.body.bytesize}B)." }
|
289
|
+
log.trace { "POST #{@api} body=#{post.body}" }
|
353
290
|
|
354
|
-
|
355
|
-
response = @
|
356
|
-
|
291
|
+
t1 = Time.now
|
292
|
+
response = @conn.request @api, post
|
293
|
+
t2 = Time.now
|
357
294
|
|
358
295
|
# raise Exception to utilize Fluentd output plugin retry machanism
|
359
296
|
raise "Server error (#{response.code}) for POST #{@hec_api}, response: #{response.body}" if response.code.start_with?('5')
|
@@ -361,9 +298,12 @@ module Fluent::Plugin
|
|
361
298
|
# For both success response (2xx) and client errors (4xx), we will consume the chunk.
|
362
299
|
# Because there probably a bug in the code if we get 4xx errors, retry won't do any good.
|
363
300
|
if not response.code.start_with?('2')
|
364
|
-
|
365
|
-
|
301
|
+
log.error "Failed POST to #{@hec_api}, response: #{response.body}"
|
302
|
+
log.debug { "Failed request body: #{post.body}" }
|
366
303
|
end
|
304
|
+
|
305
|
+
log.debug { "[Response] Chunk: #{dump_unique_id_hex(chunk.unique_id)} Size: #{post.body.bytesize} Response: #{response.inspect} Duration: #{t2 - t1}" }
|
306
|
+
process_response(response, post.body)
|
367
307
|
end
|
368
308
|
|
369
309
|
# Encode as UTF-8. If 'coerce_to_utf8' is set to true in the config, any
|
@@ -374,32 +314,32 @@ module Fluent::Plugin
|
|
374
314
|
# https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud/blob/dbc28575/lib/fluent/plugin/out_google_cloud.rb#L1284
|
375
315
|
def convert_to_utf8(input)
|
376
316
|
if input.is_a?(Hash)
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
317
|
+
record = {}
|
318
|
+
input.each do |key, value|
|
319
|
+
record[convert_to_utf8(key)] = convert_to_utf8(value)
|
320
|
+
end
|
381
321
|
|
382
|
-
|
322
|
+
return record
|
383
323
|
end
|
384
324
|
return input.map { |value| convert_to_utf8(value) } if input.is_a?(Array)
|
385
325
|
return input unless input.respond_to?(:encode)
|
386
326
|
|
387
327
|
if @coerce_to_utf8
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
328
|
+
input.encode(
|
329
|
+
'utf-8',
|
330
|
+
invalid: :replace,
|
331
|
+
undef: :replace,
|
332
|
+
replace: @non_utf8_replacement_string)
|
333
|
+
else
|
334
|
+
begin
|
335
|
+
input.encode('utf-8')
|
336
|
+
rescue EncodingError
|
337
|
+
log.error { 'Encountered encoding issues potentially due to non ' \
|
338
|
+
'UTF-8 characters. To allow non-UTF-8 characters and ' \
|
339
|
+
'replace them with spaces, please set "coerce_to_utf8" ' \
|
340
|
+
'to true.' }
|
341
|
+
raise
|
342
|
+
end
|
403
343
|
end
|
404
344
|
end
|
405
345
|
end
|