fluent-plugin-splunk-hec 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +2 -0
- data/Gemfile.lock +125 -14
- data/README.md +136 -74
- data/Rakefile +6 -1
- data/VERSION +1 -1
- data/fluent-plugin-splunk-hec.gemspec +9 -3
- data/lib/fluent/plugin/out_splunk.rb +313 -0
- data/lib/fluent/plugin/{out_splunk_hec → out_splunk}/match_formatter.rb +5 -3
- data/lib/fluent/plugin/out_splunk/version.rb +3 -0
- data/lib/fluent/plugin/out_splunk_hec.rb +130 -190
- data/lib/fluent/plugin/out_splunk_hec/version.rb +2 -0
- data/lib/fluent/plugin/out_splunk_ingest_api.rb +109 -0
- data/test/fluent/plugin/out_splunk_hec_test.rb +232 -221
- data/test/fluent/plugin/out_splunk_ingest_api_test.rb +244 -0
- data/test/test_helper.rb +10 -7
- metadata +82 -23
- data/test/lib/webmock/http_lib_adapters/httpclient_adapter.rb +0 -0
@@ -0,0 +1,109 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'fluent/plugin/out_splunk'
|
4
|
+
require 'openid_connect'
|
5
|
+
require 'rack/oauth2'
|
6
|
+
require 'multi_json'
|
7
|
+
|
8
|
+
module Fluent::Plugin
|
9
|
+
class SplunkIngestApiOutput < SplunkOutput
|
10
|
+
Fluent::Plugin.register_output('splunk_ingest_api', self)
|
11
|
+
|
12
|
+
desc 'Service Client Identifier'
|
13
|
+
config_param :service_client_identifier, :string, default: nil
|
14
|
+
|
15
|
+
desc 'Service Client Secret Key'
|
16
|
+
config_param :service_client_secret_key, :string, default: nil
|
17
|
+
|
18
|
+
desc 'Token Endpoint'
|
19
|
+
config_param :token_endpoint, :string, default: '/system/identity/v1/token'
|
20
|
+
|
21
|
+
desc 'Ingest Api Hostname'
|
22
|
+
config_param :ingest_api_host, :string, default: 'api.splunkbeta.com'
|
23
|
+
|
24
|
+
desc 'Ingest API Tenant Name'
|
25
|
+
config_param :ingest_api_tenant, :string
|
26
|
+
|
27
|
+
desc 'Ingest API Events Endpoint'
|
28
|
+
config_param :ingest_api_events_endpoint, :string, default: '/ingest/v1beta2/events'
|
29
|
+
|
30
|
+
desc 'Debug the HTTP transport'
|
31
|
+
config_param :debug_http, :bool, default: false
|
32
|
+
|
33
|
+
def prefer_buffer_processing
|
34
|
+
true
|
35
|
+
end
|
36
|
+
|
37
|
+
def configure(conf)
|
38
|
+
super
|
39
|
+
end
|
40
|
+
|
41
|
+
def construct_api
|
42
|
+
uri = "https://#{@ingest_api_host}/#{@ingest_api_tenant}#{@ingest_api_events_endpoint}"
|
43
|
+
URI(uri)
|
44
|
+
rescue StandardError
|
45
|
+
raise Fluent::ConfigError, "URI #{uri} is invalid"
|
46
|
+
end
|
47
|
+
|
48
|
+
def format(tag, time, record)
|
49
|
+
format_event(tag, time, record)
|
50
|
+
end
|
51
|
+
|
52
|
+
def format_event(tag, time, record)
|
53
|
+
event = prepare_event_payload(tag, time, record)
|
54
|
+
# Unsure how to drop a record. So append the empty string
|
55
|
+
if event[:body].nil? || event[:body].strip.empty?
|
56
|
+
''
|
57
|
+
else
|
58
|
+
MultiJson.dump(event) + ','
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def prepare_event_payload(tag, time, record)
|
63
|
+
payload = super(tag, time, record)
|
64
|
+
payload[:attributes] = payload.delete(:fields) || {}
|
65
|
+
payload[:attributes][:index] = payload.delete(:index) if payload[:index]
|
66
|
+
payload[:body] = payload.delete(:event)
|
67
|
+
payload.delete(:time)
|
68
|
+
payload[:timestamp] = (time.to_f * 1000).to_i
|
69
|
+
payload[:nanos] = time.nsec / 100_000
|
70
|
+
|
71
|
+
payload
|
72
|
+
end
|
73
|
+
|
74
|
+
def process_response(response, request_body)
|
75
|
+
super
|
76
|
+
if response.code.to_s == '401'
|
77
|
+
@conn = new_connection
|
78
|
+
raise 'Auth Error recived. New token has been fetched.'
|
79
|
+
elsif response.code.to_s == '429'
|
80
|
+
raise "Throttle error from server. #{response.body}"
|
81
|
+
elsif /INVALID_DATA/.match?(response.body)
|
82
|
+
log.error "#{self.class}: POST Body #{request_body}"
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def new_connection
|
87
|
+
Rack::OAuth2.debugging = true if @debug_http
|
88
|
+
client = OpenIDConnect::Client.new(
|
89
|
+
token_endpoint: @token_endpoint,
|
90
|
+
identifier: @service_client_identifier,
|
91
|
+
secret: @service_client_secret_key,
|
92
|
+
redirect_uri: 'http://localhost:8080/', # Not used
|
93
|
+
host: @ingest_api_host,
|
94
|
+
scheme: 'https'
|
95
|
+
)
|
96
|
+
|
97
|
+
client.access_token!(client_auth_method: 'other')
|
98
|
+
end
|
99
|
+
|
100
|
+
def write_to_splunk(chunk)
|
101
|
+
log.trace "#{self.class}: In write() with #{chunk.size_of_events} records and #{chunk.bytesize} bytes "
|
102
|
+
# ingest API is an array of json objects
|
103
|
+
body = "[#{chunk.read.chomp(',')}]"
|
104
|
+
@conn ||= new_connection
|
105
|
+
response = @conn.post("https://#{@ingest_api_host}/#{@ingest_api_tenant}#{@ingest_api_events_endpoint}", body: body)
|
106
|
+
process_response(response, body)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
@@ -1,92 +1,103 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'test_helper'
|
2
4
|
|
3
5
|
describe Fluent::Plugin::SplunkHecOutput do
|
4
6
|
include Fluent::Test::Helpers
|
5
7
|
include PluginTestHelper
|
6
8
|
|
7
9
|
before { Fluent::Test.setup } # setup router and others
|
8
|
-
|
10
|
+
|
9
11
|
it { expect(::Fluent::Plugin::SplunkHecOutput::VERSION).wont_be_nil }
|
10
12
|
|
11
|
-
describe
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
13
|
+
describe 'config param tests' do
|
14
|
+
it 'should require https protocol' do
|
15
|
+
expect(create_hec_output_driver('hec_host protocol').instance.protocol).must_equal :https
|
16
|
+
end
|
17
|
+
it 'should require hec_host' do
|
18
|
+
expect(create_hec_output_driver('hec_host hec_host').instance.hec_host).must_equal 'hec_host'
|
19
|
+
end
|
20
|
+
it 'should require hec_port' do
|
21
|
+
expect(create_hec_output_driver('hec_host hec_port').instance.hec_port).must_equal 8088
|
22
|
+
end
|
23
|
+
it 'should require hec_token' do
|
24
|
+
expect(create_hec_output_driver('hec_host hec_token').instance.hec_token).must_equal 'some-token'
|
25
|
+
end
|
26
|
+
it 'should define client_cert as nil initially' do
|
27
|
+
assert_nil(create_hec_output_driver('hec_host hec_token').instance.client_cert)
|
28
|
+
end
|
29
|
+
it 'should define client_key as nil (string) initially' do
|
30
|
+
assert_nil(create_hec_output_driver('hec_host hec_token').instance.client_key)
|
31
|
+
expect(create_hec_output_driver('hec_host hec_token').instance.client_key).is_a? String
|
32
|
+
end
|
33
|
+
it 'should define ca_file as nil (string) initially' do
|
34
|
+
assert_nil(create_hec_output_driver('hec_host hec_token').instance.ca_file)
|
35
|
+
expect(create_hec_output_driver('hec_host hec_token').instance.ca_file).is_a? String
|
36
|
+
end
|
37
|
+
it 'should define ca_path as nil (string) initially' do
|
38
|
+
assert_nil(create_hec_output_driver('hec_host hec_token').instance.ca_path)
|
39
|
+
expect(create_hec_output_driver('hec_host hec_token').instance.ca_path).is_a? String
|
40
|
+
end
|
41
|
+
it 'should define ssl_ciphers as nil (array) initially' do
|
42
|
+
assert_nil(create_hec_output_driver('hec_host hec_token').instance.ssl_ciphers)
|
43
|
+
expect(create_hec_output_driver('hec_host hec_token').instance.ssl_ciphers).is_a? Array
|
44
|
+
end
|
45
|
+
it 'should not allow an insecure ssl connection' do
|
46
|
+
expect(create_hec_output_driver('hec_host hec_token').instance.insecure_ssl).must_equal false
|
47
|
+
end
|
48
|
+
it 'should allow both event (default) and metric to be sent to splunk' do
|
49
|
+
expect(create_hec_output_driver('hec_host hec_token').instance.data_type).must_equal :event
|
50
|
+
expect(create_hec_output_driver('hec_host hec_token').instance.data_type = :metric).must_equal :metric
|
51
|
+
end
|
52
|
+
it 'should define Splunk index to index (string) as nil initially' do
|
53
|
+
assert_nil(create_hec_output_driver('hec_host hec_token').instance.index)
|
54
|
+
expect(create_hec_output_driver('hec_host hec_token').instance.index).is_a? String
|
55
|
+
end
|
56
|
+
it 'should define field names to include Splunk index_key as nil (string) initially' do
|
57
|
+
assert_nil(create_hec_output_driver('hec_host hec_token').instance.index_key)
|
58
|
+
expect(create_hec_output_driver('hec_host hec_token').instance.index_key).is_a? String
|
59
|
+
end
|
58
60
|
end
|
59
61
|
|
60
|
-
describe
|
61
|
-
describe
|
62
|
-
it
|
63
|
-
|
62
|
+
describe 'hec_host validation' do
|
63
|
+
describe 'invalid host' do
|
64
|
+
it 'should require hec_host' do
|
65
|
+
expect { create_hec_output_driver }.must_raise Fluent::ConfigError
|
64
66
|
end
|
65
67
|
|
66
|
-
it { expect{
|
68
|
+
it { expect { create_hec_output_driver('hec_host %bad-host%') }.must_raise Fluent::ConfigError }
|
67
69
|
end
|
68
70
|
|
69
|
-
describe
|
71
|
+
describe 'good host' do
|
70
72
|
it {
|
71
|
-
|
73
|
+
expect(create_hec_output_driver('hec_host splunk.com').instance.hec_host).must_equal 'splunk.com'
|
72
74
|
}
|
73
75
|
end
|
74
76
|
end
|
75
77
|
|
76
|
-
it
|
77
|
-
req = verify_sent_events
|
78
|
+
it 'should send request to Splunk' do
|
79
|
+
req = verify_sent_events do |batch|
|
78
80
|
expect(batch.size).must_equal 2
|
79
|
-
|
81
|
+
end
|
80
82
|
expect(req).must_be_requested times: 1
|
81
83
|
end
|
82
84
|
|
83
|
-
it
|
84
|
-
verify_sent_events
|
85
|
+
it 'should use string for event time, and the value of the string should be a float' do
|
86
|
+
verify_sent_events do |batch|
|
85
87
|
batch.each do |item|
|
86
88
|
expect(item['time']).must_be_instance_of String
|
87
89
|
expect(item['time']).must_match /^\d+\.\d+$/
|
88
90
|
end
|
89
|
-
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
# it "should contain splunk event time field via fluentd, as nil" do
|
95
|
+
# expect(create_hec_output_driver('hec_host splunk.com').instance.time_key).must_equal nil
|
96
|
+
# end
|
97
|
+
#
|
98
|
+
it 'should contain splunk event time field via fluentd, as nil' do
|
99
|
+
test_driver = create_hec_output_driver('hec_host splunk.com')
|
100
|
+
assert_nil(test_driver.instance.time_key)
|
90
101
|
end
|
91
102
|
|
92
103
|
# it "should contain splunk event time field via fluentd, as nil" do
|
@@ -99,236 +110,236 @@ describe Fluent::Plugin::SplunkHecOutput do
|
|
99
110
|
end
|
100
111
|
|
101
112
|
it "should use host machine's hostname for event host by default" do
|
102
|
-
verify_sent_events
|
113
|
+
verify_sent_events do |batch|
|
103
114
|
batch.each do |item|
|
104
|
-
|
115
|
+
expect(item['host']).must_equal Socket.gethostname
|
105
116
|
end
|
106
|
-
|
117
|
+
end
|
107
118
|
end
|
108
119
|
|
109
120
|
%w[index source sourcetype].each do |field|
|
110
121
|
it "should not set #{field} by default" do
|
111
|
-
verify_sent_events
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
122
|
+
verify_sent_events do |batch|
|
123
|
+
batch.each do |item|
|
124
|
+
expect(item).wont_include field
|
125
|
+
end
|
126
|
+
end
|
116
127
|
end
|
117
128
|
end
|
118
129
|
|
119
|
-
it
|
120
|
-
verify_sent_events(<<~CONF)
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
130
|
+
it 'should support ${tag}' do
|
131
|
+
verify_sent_events(<<~CONF) do |batch|
|
132
|
+
index ${tag}
|
133
|
+
host ${tag}
|
134
|
+
source ${tag}
|
135
|
+
sourcetype ${tag}
|
125
136
|
CONF
|
126
137
|
batch.each do |item|
|
127
|
-
|
128
|
-
|
129
|
-
|
138
|
+
%w[index host source sourcetype].each do |field|
|
139
|
+
expect(%w[tag.event1 tag.event2]).must_include item[field]
|
140
|
+
end
|
130
141
|
end
|
131
|
-
|
142
|
+
end
|
132
143
|
end
|
133
144
|
|
134
|
-
it
|
135
|
-
verify_sent_events(<<~CONF)
|
145
|
+
it 'should support *_key' do
|
146
|
+
verify_sent_events(<<~CONF) do |batch|
|
136
147
|
index_key level
|
137
148
|
host_key from
|
138
149
|
source_key file
|
139
150
|
sourcetype_key agent.name
|
140
151
|
CONF
|
141
|
-
batch.each
|
152
|
+
batch.each do |item|
|
142
153
|
expect(item['index']).must_equal 'info'
|
143
154
|
expect(item['host']).must_equal 'my_machine'
|
144
155
|
expect(item['source']).must_equal 'cool.log'
|
145
156
|
expect(item['sourcetype']).must_equal 'test'
|
146
157
|
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
158
|
+
JSON.load(item['event']).tap do |event|
|
159
|
+
%w[level from file].each { |field| expect(event).wont_include field }
|
160
|
+
expect(event['agent']).wont_include 'name'
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
153
164
|
end
|
154
165
|
|
155
|
-
it
|
156
|
-
verify_sent_events(<<~CONF)
|
166
|
+
it 'should remove nil fileds.' do
|
167
|
+
verify_sent_events(<<~CONF) do |batch|
|
157
168
|
index_key nonexist
|
158
169
|
host_key nonexist
|
159
170
|
source_key nonexist
|
160
171
|
sourcetype_key nonexist
|
161
172
|
CONF
|
162
|
-
batch.each
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
173
|
+
batch.each do |item|
|
174
|
+
expect(item).wont_be :has_key?, 'index'
|
175
|
+
expect(item).wont_be :has_key?, 'host'
|
176
|
+
expect(item).wont_be :has_key?, 'source'
|
177
|
+
expect(item).wont_be :has_key?, 'sourcetype'
|
178
|
+
end
|
179
|
+
end
|
169
180
|
end
|
170
181
|
|
171
182
|
describe 'formatter' do
|
172
|
-
it
|
173
|
-
verify_sent_events(<<~CONF)
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
183
|
+
it 'should support replace the default json formater' do
|
184
|
+
verify_sent_events(<<~CONF) do |batch|
|
185
|
+
<format>
|
186
|
+
@type single_value
|
187
|
+
message_key log
|
188
|
+
add_newline false
|
189
|
+
</format>
|
179
190
|
CONF
|
180
|
-
|
181
|
-
|
182
|
-
|
191
|
+
batch.map { |item| item['event'] }
|
192
|
+
.each { |event| expect(event).must_equal 'everything is good' }
|
193
|
+
end
|
183
194
|
end
|
184
195
|
|
185
|
-
it
|
186
|
-
verify_sent_events(<<~CONF)
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
196
|
+
it 'should support multiple formatters' do
|
197
|
+
verify_sent_events(<<~CONF) do |batch|
|
198
|
+
source ${tag}
|
199
|
+
<format tag.event1>
|
200
|
+
@type single_value
|
201
|
+
message_key log
|
202
|
+
add_newline false
|
203
|
+
</format>
|
193
204
|
CONF
|
194
|
-
|
195
|
-
|
196
|
-
|
205
|
+
expect(batch.find { |item| item['source'] == 'tag.event1' }['event']).must_equal 'everything is good'
|
206
|
+
expect(batch.find { |item| item['source'] == 'tag.event2' }['event']).must_be_instance_of Hash
|
207
|
+
end
|
197
208
|
end
|
198
209
|
end
|
199
210
|
|
200
|
-
it
|
201
|
-
verify_sent_events(<<~CONF)
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
211
|
+
it 'should support fields for indexed field extraction' do
|
212
|
+
verify_sent_events(<<~CONF) do |batch|
|
213
|
+
<fields>
|
214
|
+
from
|
215
|
+
logLevel level
|
216
|
+
nonexist
|
217
|
+
</fields>
|
207
218
|
CONF
|
208
219
|
batch.each do |item|
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
220
|
+
JSON.load(item['event']).tap do |event|
|
221
|
+
expect(event).wont_include 'from'
|
222
|
+
expect(event).wont_include 'level'
|
223
|
+
end
|
224
|
+
|
225
|
+
expect(item['fields']['from']).must_equal 'my_machine'
|
226
|
+
expect(item['fields']['logLevel']).must_equal 'info'
|
227
|
+
expect(item['fields']).wont_be :has_key?, 'nonexist'
|
217
228
|
end
|
218
|
-
|
229
|
+
end
|
219
230
|
end
|
220
231
|
|
221
|
-
describe 'metric'do
|
232
|
+
describe 'metric' do
|
222
233
|
it 'should check related configs' do
|
223
234
|
expect(
|
224
|
-
|
235
|
+
create_hec_output_driver('hec_host somehost', 'data_type metric')
|
225
236
|
).wont_be_nil
|
226
237
|
|
227
|
-
expect
|
228
|
-
|
229
|
-
|
238
|
+
expect do
|
239
|
+
create_hec_output_driver('hec_host somehost', 'data_type metric', 'metrics_from_event false')
|
240
|
+
end.must_raise Fluent::ConfigError
|
230
241
|
|
231
|
-
expect
|
232
|
-
|
233
|
-
|
242
|
+
expect do
|
243
|
+
create_hec_output_driver('hec_host somehost', 'data_type metric', 'metric_name_key x')
|
244
|
+
end.must_raise Fluent::ConfigError
|
234
245
|
|
235
246
|
expect(
|
236
|
-
|
247
|
+
create_hec_output_driver('hec_host somehost', 'data_type metric', 'metric_name_key x', 'metric_value_key y')
|
237
248
|
).wont_be_nil
|
238
249
|
end
|
239
250
|
|
240
251
|
it 'should have "metric" as event, and have proper fields' do
|
241
|
-
verify_sent_events(<<~CONF)
|
242
|
-
|
243
|
-
|
244
|
-
|
252
|
+
verify_sent_events(<<~CONF) do |batch|
|
253
|
+
data_type metric
|
254
|
+
metric_name_key from
|
255
|
+
metric_value_key value
|
245
256
|
CONF
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
257
|
+
batch.each do |item|
|
258
|
+
expect(item['event']).must_equal 'metric'
|
259
|
+
expect(item['fields']['metric_name']).must_equal 'my_machine'
|
260
|
+
expect(item['fields']['_value']).must_equal 100
|
261
|
+
expect(item['fields']['log']).must_equal 'everything is good'
|
262
|
+
expect(item['fields']['level']).must_equal 'info'
|
263
|
+
expect(item['fields']['file']).must_equal 'cool.log'
|
264
|
+
end
|
265
|
+
end
|
255
266
|
end
|
256
267
|
|
257
268
|
it 'should handle empty fields' do
|
258
|
-
verify_sent_events(<<~CONF)
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
269
|
+
verify_sent_events(<<~CONF) do |batch|
|
270
|
+
data_type metric
|
271
|
+
metric_name_key from
|
272
|
+
metric_value_key value
|
273
|
+
<fields>
|
274
|
+
</fields>
|
264
275
|
CONF
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
276
|
+
batch.each do |item|
|
277
|
+
# only "metric_name" and "_value"
|
278
|
+
expect(item['fields'].keys.size).must_equal 2
|
279
|
+
end
|
280
|
+
end
|
270
281
|
end
|
271
282
|
|
272
283
|
it 'should handle custom fields' do
|
273
|
-
verify_sent_events(<<~CONF)
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
284
|
+
verify_sent_events(<<~CONF) do |batch|
|
285
|
+
data_type metric
|
286
|
+
metric_name_key from
|
287
|
+
metric_value_key value
|
288
|
+
<fields>
|
289
|
+
level
|
290
|
+
filePath file
|
291
|
+
username
|
292
|
+
</fields>
|
282
293
|
CONF
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
294
|
+
batch.each do |item|
|
295
|
+
expect(item['fields'].keys.size).must_equal 4
|
296
|
+
expect(item['fields']['level']).must_equal 'info'
|
297
|
+
expect(item['fields']['filePath']).must_equal 'cool.log'
|
298
|
+
# null fields should be removed
|
299
|
+
expect(item['fields']).wont_be :has_key?, 'username'
|
300
|
+
end
|
301
|
+
end
|
291
302
|
end
|
292
303
|
|
293
304
|
it 'should treat each key-value in event as a metric' do
|
294
305
|
metrics = [
|
295
|
-
['tag', event_time, {'cup': 0.5, 'memory': 100}],
|
296
|
-
['tag', event_time, {'cup': 0.6, 'memory': 200}]
|
306
|
+
['tag', event_time, { 'cup': 0.5, 'memory': 100 }],
|
307
|
+
['tag', event_time, { 'cup': 0.6, 'memory': 200 }]
|
297
308
|
]
|
298
|
-
with_stub_hec(events: metrics, conf: 'data_type metric')
|
299
|
-
|
300
|
-
|
309
|
+
with_stub_hec(events: metrics, conf: 'data_type metric') do |batch|
|
310
|
+
expect(batch.size).must_equal 4
|
311
|
+
end
|
301
312
|
end
|
302
313
|
end
|
303
314
|
|
304
315
|
describe 'timeout params' do
|
305
|
-
|
306
|
-
|
307
|
-
|
316
|
+
it 'should reset unused connection after 5 seconds' do
|
317
|
+
expect(create_hec_output_driver('hec_host splunk.com', 'idle_timeout 5').instance.idle_timeout).must_equal 5
|
318
|
+
end
|
308
319
|
|
309
|
-
|
310
|
-
|
311
|
-
|
320
|
+
it 'should allow custom setting between reading chunks from the socket' do
|
321
|
+
expect(create_hec_output_driver('hec_host splunk.com', 'read_timeout 5').instance.read_timeout).must_equal 5
|
322
|
+
end
|
312
323
|
|
313
|
-
|
314
|
-
|
315
|
-
|
324
|
+
it 'should allow custom setting a connection to be opened' do
|
325
|
+
expect(create_hec_output_driver('hec_host splunk.com', 'open_timeout 5').instance.open_timeout).must_equal 5
|
326
|
+
end
|
316
327
|
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
end
|
328
|
+
it 'should check default values are created correctly for timeout params' do
|
329
|
+
test_driver = create_hec_output_driver('hec_host splunk.com')
|
330
|
+
expect(test_driver.instance.idle_timeout).must_equal 5
|
331
|
+
assert_nil(test_driver.instance.read_timeout)
|
332
|
+
assert_nil(test_driver.instance.open_timeout)
|
323
333
|
end
|
334
|
+
end
|
324
335
|
|
325
|
-
def with_stub_hec(events:, conf: ''
|
326
|
-
host =
|
327
|
-
@driver =
|
336
|
+
def with_stub_hec(events:, conf: '')
|
337
|
+
host = 'hec.splunk.com'
|
338
|
+
@driver = create_hec_output_driver("hec_host #{host}", conf)
|
328
339
|
|
329
|
-
hec_req = stub_hec_request("https://#{host}:8088").with
|
330
|
-
|
331
|
-
|
340
|
+
hec_req = stub_hec_request("https://#{host}:8088").with do |r|
|
341
|
+
yield r.body.split(/(?={)\s*(?<=})/).map { |item| JSON.load item }
|
342
|
+
end
|
332
343
|
|
333
344
|
@driver.run do
|
334
345
|
events.each { |evt| @driver.feed *evt }
|
@@ -339,19 +350,19 @@ describe Fluent::Plugin::SplunkHecOutput do
|
|
339
350
|
|
340
351
|
def verify_sent_events(conf = '', &blk)
|
341
352
|
event = {
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
353
|
+
'log' => 'everything is good',
|
354
|
+
'level' => 'info',
|
355
|
+
'from' => 'my_machine',
|
356
|
+
'file' => 'cool.log',
|
357
|
+
'value' => 100,
|
358
|
+
'agent' => {
|
359
|
+
'name' => 'test',
|
360
|
+
'version' => '1.0.0'
|
350
361
|
}
|
351
362
|
}
|
352
363
|
events = [
|
353
|
-
[
|
354
|
-
[
|
364
|
+
['tag.event1', event_time, { 'id' => '1st' }.merge(Marshal.load(Marshal.dump(event)))],
|
365
|
+
['tag.event2', event_time, { 'id' => '2nd' }.merge(Marshal.load(Marshal.dump(event)))]
|
355
366
|
]
|
356
367
|
|
357
368
|
with_stub_hec conf: conf, events: events, &blk
|