fluent-plugin-splunkhec 1.1 → 1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +5 -0
- data/fluent-plugin-splunkhec.gemspec +2 -1
- data/lib/fluent/plugin/out_splunkhec.rb +60 -66
- data/test/plugin/test_out_splunkhec.rb +151 -14
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b9180a1fe84fc9e92e0bd1cc7f190030fc66ac82
|
4
|
+
data.tar.gz: 14530543bf7ad149b53c698ffde814717fa87ffa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6d9c296da32fa4c7c5905e62d912a9459fbdefb5f513c92b19caac58f08630ebba80552f6d36474359edfb380c8093d0adb5e43d1d532fda335cab7757af70ba
|
7
|
+
data.tar.gz: b473a7be07fd74d4f50ab81218a8e9dc07cfb598f70637fdb9e5a0c25d8361c0ee199382b90af3964dd097e281e311dd21df2f150ee670849935854324df7155
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -23,6 +23,7 @@ The Splunk HEC is running on a Heavy Forwarder or single instance. More info abo
|
|
23
23
|
sourcetype data:type #optional
|
24
24
|
usejson true #optional defaults to true
|
25
25
|
send_event_as_json true #optional
|
26
|
+
send_batched_events false #optional
|
26
27
|
</source>
|
27
28
|
```
|
28
29
|
|
@@ -67,6 +68,10 @@ Specify if an event should be sent as json rather than as a string. Can be 'true
|
|
67
68
|
|
68
69
|
Specify the event type as JSON (true|default) or raw (false) for sending Log4J messages so Splunk so it can parse the time field it self based on the format 'time' regex match found in the source, uses millisecond precision.
|
69
70
|
|
71
|
+
## config: send_batched_events
|
72
|
+
|
73
|
+
Specify that all events in a FluentD chunk should be sent in batch to Splunk. Defaults to 'false' which sends one event at a time. Batching events will reduce the load on the Splunk HEC. Max chunk size is controlled by config parameter 'buffer_chunk_limit' and should be matched by the Splunk limit 'max_content_length'. Please see this [blog post](https://www.splunk.com/blog/2016/08/12/handling-http-event-collector-hec-content-length-too-large-errors-without-pulling-your-hair-out.html) for details.
|
74
|
+
|
70
75
|
## Contributing
|
71
76
|
|
72
77
|
1. Fork it
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = "fluent-plugin-splunkhec"
|
7
|
-
gem.version = "1.
|
7
|
+
gem.version = "1.2"
|
8
8
|
gem.authors = "Coen Meerbeek"
|
9
9
|
gem.email = "cmeerbeek@gmail.com"
|
10
10
|
gem.description = %q{Output plugin for the Splunk HTTP Event Collector.}
|
@@ -20,5 +20,6 @@ Gem::Specification.new do |gem|
|
|
20
20
|
gem.add_dependency "json", '~> 2.0', '>= 2.0.2'
|
21
21
|
gem.add_development_dependency "rake", '~> 0.9', '>= 0.9.2'
|
22
22
|
gem.add_development_dependency "test-unit", '~> 3.1', '>= 3.1.0'
|
23
|
+
gem.add_development_dependency "webmock", '>= 3.0'
|
23
24
|
gem.license = 'MIT'
|
24
25
|
end
|
@@ -10,51 +10,32 @@ module Fluent
|
|
10
10
|
config_param :host, :string, :default => 'localhost'
|
11
11
|
config_param :protocol, :string, :default => 'http'
|
12
12
|
config_param :port, :string, :default => '8088'
|
13
|
-
config_param :token, :string
|
13
|
+
config_param :token, :string
|
14
14
|
|
15
15
|
# Splunk event parameters
|
16
|
-
config_param :index,
|
17
|
-
config_param :event_host,
|
18
|
-
config_param :source,
|
19
|
-
config_param :sourcetype,
|
20
|
-
config_param :send_event_as_json,
|
21
|
-
config_param :usejson,
|
16
|
+
config_param :index, :string, :default => 'main'
|
17
|
+
config_param :event_host, :string, :default => nil
|
18
|
+
config_param :source, :string, :default => 'fluentd'
|
19
|
+
config_param :sourcetype, :string, :default => 'tag'
|
20
|
+
config_param :send_event_as_json, :bool, :default => false
|
21
|
+
config_param :usejson, :bool, :default => true
|
22
|
+
config_param :send_batched_events, :bool, :default => false
|
22
23
|
|
23
24
|
# This method is called before starting.
|
24
25
|
# Here we construct the Splunk HEC URL to POST data to
|
25
26
|
# If the configuration is invalid, raise Fluent::ConfigError.
|
26
27
|
def configure(conf)
|
27
28
|
super
|
28
|
-
|
29
29
|
@splunk_url = @protocol + '://' + @host + ':' + @port + '/services/collector/event'
|
30
30
|
log.debug 'splunkhec: sent data to ' + @splunk_url
|
31
|
-
if conf['token'] != nil
|
32
|
-
@token = conf['token']
|
33
|
-
else
|
34
|
-
raise 'splunkhec: token is empty, please provide a token for this plugin to work'
|
35
|
-
end
|
36
31
|
|
37
32
|
if conf['event_host'] == nil
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
if conf['sourcetype'] == nil
|
45
|
-
@event_sourcetype = 'tag'
|
46
|
-
else
|
47
|
-
@event_sourcetype = conf['sourcetype']
|
33
|
+
begin
|
34
|
+
@event_host = `hostname`.delete!("\n")
|
35
|
+
rescue
|
36
|
+
@event_host = 'unknown'
|
37
|
+
end
|
48
38
|
end
|
49
|
-
|
50
|
-
if conf['send_event_as_json'] == 'true'
|
51
|
-
@event_send_as_json = true
|
52
|
-
else
|
53
|
-
@event_send_as_json = false
|
54
|
-
end
|
55
|
-
|
56
|
-
@event_index = @index
|
57
|
-
@event_source = @source
|
58
39
|
end
|
59
40
|
|
60
41
|
def start
|
@@ -74,13 +55,14 @@ module Fluent
|
|
74
55
|
# Loop through all records and sent them to Splunk
|
75
56
|
def write(chunk)
|
76
57
|
begin
|
58
|
+
body = ''
|
77
59
|
chunk.msgpack_each {|(tag,time,record)|
|
78
60
|
# Parse record to Splunk event format
|
79
61
|
case record
|
80
62
|
when Fixnum
|
81
63
|
event = record.to_s
|
82
64
|
when Hash
|
83
|
-
if @
|
65
|
+
if @send_event_as_json
|
84
66
|
event = record.to_json
|
85
67
|
else
|
86
68
|
event = record.to_json.gsub("\"", %q(\\\"))
|
@@ -89,51 +71,63 @@ module Fluent
|
|
89
71
|
event = record
|
90
72
|
end
|
91
73
|
|
92
|
-
|
93
|
-
@event_sourcetype = tag
|
94
|
-
end
|
74
|
+
sourcetype = @sourcetype == 'tag' ? tag : @sourcetype
|
95
75
|
|
96
76
|
# Build body for the POST request
|
97
|
-
if
|
77
|
+
if !@usejson
|
98
78
|
event = record["time"]+ " " + record["message"].to_json.gsub(/^"|"$/,"")
|
99
|
-
body
|
100
|
-
elsif @
|
101
|
-
body
|
79
|
+
body << '{"time":"'+ DateTime.parse(record["time"]).strftime("%Q") +'", "event":"' + event + '", "sourcetype" :"' + sourcetype + '", "source" :"' + @source + '", "index" :"' + @index + '", "host" : "' + @event_host + '"}'
|
80
|
+
elsif @send_event_as_json
|
81
|
+
body << '{"time" :' + time.to_s + ', "event" :' + event + ', "sourcetype" :"' + sourcetype + '", "source" :"' + @source + '", "index" :"' + @index + '", "host" : "' + @event_host + '"}'
|
102
82
|
else
|
103
|
-
body
|
104
|
-
end
|
105
|
-
log.debug "splunkhec: " + body + "\n"
|
106
|
-
|
107
|
-
uri = URI(@splunk_url)
|
108
|
-
|
109
|
-
# Create client
|
110
|
-
http = Net::HTTP.new(uri.host, uri.port)
|
111
|
-
|
112
|
-
# Create Request
|
113
|
-
req = Net::HTTP::Post.new(uri)
|
114
|
-
# Add headers
|
115
|
-
req.add_field "Authorization", "Splunk #{@token}"
|
116
|
-
# Add headers
|
117
|
-
req.add_field "Content-Type", "application/json; charset=utf-8"
|
118
|
-
# Set body
|
119
|
-
req.body = body
|
120
|
-
# Handle SSL
|
121
|
-
if @protocol == 'https'
|
122
|
-
http.use_ssl = true
|
123
|
-
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
83
|
+
body << '{"time" :' + time.to_s + ', "event" :"' + event + '", "sourcetype" :"' + sourcetype + '", "source" :"' + @source + '", "index" :"' + @index + '", "host" : "' + @event_host + '"}'
|
124
84
|
end
|
125
85
|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
86
|
+
if @send_batched_events
|
87
|
+
body << "\n"
|
88
|
+
else
|
89
|
+
send_to_splunk(body)
|
90
|
+
body = ''
|
131
91
|
end
|
132
92
|
}
|
93
|
+
|
94
|
+
if @send_batched_events
|
95
|
+
send_to_splunk(body)
|
96
|
+
end
|
133
97
|
rescue => err
|
134
98
|
log.fatal("splunkhec: caught exception; exiting")
|
135
99
|
log.fatal(err)
|
136
100
|
end
|
137
101
|
end
|
102
|
+
|
103
|
+
def send_to_splunk(body)
|
104
|
+
log.debug "splunkhec: " + body + "\n"
|
105
|
+
|
106
|
+
uri = URI(@splunk_url)
|
107
|
+
|
108
|
+
# Create client
|
109
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
110
|
+
|
111
|
+
# Create Request
|
112
|
+
req = Net::HTTP::Post.new(uri)
|
113
|
+
# Add headers
|
114
|
+
req.add_field "Authorization", "Splunk #{@token}"
|
115
|
+
# Add headers
|
116
|
+
req.add_field "Content-Type", "application/json; charset=utf-8"
|
117
|
+
# Set body
|
118
|
+
req.body = body
|
119
|
+
# Handle SSL
|
120
|
+
if @protocol == 'https'
|
121
|
+
http.use_ssl = true
|
122
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
123
|
+
end
|
124
|
+
|
125
|
+
# Fetch Request
|
126
|
+
res = http.request(req)
|
127
|
+
log.debug "splunkhec: response HTTP Status Code is #{res.code}"
|
128
|
+
if res.code.to_i != 200
|
129
|
+
log.debug "splunkhec: response body is #{res.body}"
|
130
|
+
end
|
131
|
+
end
|
138
132
|
end
|
139
133
|
end
|
@@ -1,28 +1,165 @@
|
|
1
1
|
require 'helper'
|
2
|
+
require 'webmock/test_unit'
|
3
|
+
|
2
4
|
|
3
5
|
class SplunkHECOutputTest < Test::Unit::TestCase
|
6
|
+
HOST = 'splunk.example.com'
|
7
|
+
PROTOCOL = 'https'
|
8
|
+
PORT = '8443'
|
9
|
+
TOKEN = 'BAB747F3-744E-41BA'
|
10
|
+
SOURCE = 'fluentd'
|
11
|
+
INDEX = 'main'
|
12
|
+
EVENT_HOST = 'some_host'
|
13
|
+
SOURCETYPE = 'log'
|
14
|
+
|
15
|
+
SPLUNK_URL = "#{PROTOCOL}://#{HOST}:#{PORT}/services/collector/event"
|
16
|
+
|
17
|
+
### for Splunk HEC
|
18
|
+
CONFIG = %[
|
19
|
+
host #{HOST}
|
20
|
+
protocol #{PROTOCOL}
|
21
|
+
port #{PORT}
|
22
|
+
token #{TOKEN}
|
23
|
+
source #{SOURCE}
|
24
|
+
index #{INDEX}
|
25
|
+
event_host #{EVENT_HOST}
|
26
|
+
]
|
27
|
+
|
28
|
+
def create_driver_splunkhec(conf = CONFIG)
|
29
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::SplunkHECOutput).configure(conf)
|
30
|
+
end
|
31
|
+
|
4
32
|
def setup
|
5
33
|
Fluent::Test.setup
|
34
|
+
require 'fluent/plugin/out_splunkhec'
|
35
|
+
stub_request(:any, SPLUNK_URL)
|
6
36
|
end
|
7
37
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
token BAB747F3-744E-41BA
|
14
|
-
]
|
38
|
+
def test_should_require_mandatory_parameter_token
|
39
|
+
assert_raise Fluent::ConfigError do
|
40
|
+
create_driver_splunkhec(%[])
|
41
|
+
end
|
42
|
+
end
|
15
43
|
|
16
|
-
def
|
17
|
-
|
44
|
+
def test_should_use_default_values_for_optional_parameters
|
45
|
+
d = create_driver_splunkhec(%[token some_token])
|
46
|
+
assert_equal 'localhost', d.instance.host
|
47
|
+
assert_equal 'http', d.instance.protocol
|
48
|
+
assert_equal '8088', d.instance.port
|
49
|
+
assert_equal 'main', d.instance.index
|
50
|
+
assert_equal `hostname`.delete!("\n"), d.instance.event_host
|
51
|
+
assert_equal 'fluentd', d.instance.source
|
52
|
+
assert_equal 'tag', d.instance.sourcetype
|
53
|
+
assert_equal false, d.instance.send_event_as_json
|
54
|
+
assert_equal true, d.instance.usejson
|
55
|
+
assert_equal false, d.instance.send_batched_events
|
56
|
+
assert_equal 'some_token', d.instance.token
|
18
57
|
end
|
19
58
|
|
20
|
-
def
|
59
|
+
def test_should_configure_splunkhec
|
21
60
|
d = create_driver_splunkhec
|
22
|
-
assert_equal
|
23
|
-
assert_equal
|
24
|
-
assert_equal
|
25
|
-
assert_equal
|
61
|
+
assert_equal HOST, d.instance.host
|
62
|
+
assert_equal PROTOCOL, d.instance.protocol
|
63
|
+
assert_equal PORT, d.instance.port
|
64
|
+
assert_equal TOKEN, d.instance.token
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_should_post_formatted_event_to_splunk
|
68
|
+
sourcetype = 'log'
|
69
|
+
time = 123456
|
70
|
+
record = {'message' => 'data'}
|
71
|
+
|
72
|
+
splunk_request = stub_request(:post, SPLUNK_URL)
|
73
|
+
.with(
|
74
|
+
headers: {
|
75
|
+
'Authorization' => "Splunk #{TOKEN}",
|
76
|
+
'Content-Type' => 'application/json; charset=utf-8'
|
77
|
+
},
|
78
|
+
body: {
|
79
|
+
'time' => time,
|
80
|
+
'event' => record.to_json,
|
81
|
+
'sourcetype' => sourcetype,
|
82
|
+
'source' => SOURCE,
|
83
|
+
'index' => INDEX,
|
84
|
+
'host' => EVENT_HOST
|
85
|
+
})
|
86
|
+
|
87
|
+
d = create_driver_splunkhec(CONFIG + %[sourcetype #{sourcetype}])
|
88
|
+
d.run do
|
89
|
+
d.emit(record, time)
|
90
|
+
end
|
91
|
+
|
92
|
+
assert_requested(splunk_request)
|
93
|
+
end
|
94
|
+
|
95
|
+
def test_should_use_tag_as_sourcetype_when_configured
|
96
|
+
splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'sourcetype' => 'test'}))
|
97
|
+
|
98
|
+
d = create_driver_splunkhec(CONFIG + %[sourcetype tag])
|
99
|
+
d.run do
|
100
|
+
d.emit({'message' => 'data'}, 123456)
|
101
|
+
end
|
102
|
+
|
103
|
+
assert_requested(splunk_request)
|
104
|
+
end
|
105
|
+
|
106
|
+
def test_should_send_event_as_string_as_default
|
107
|
+
record = {'message' => 'data'}
|
108
|
+
splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'event' => record.to_json}))
|
109
|
+
|
110
|
+
d = create_driver_splunkhec(CONFIG + %[send_event_as_json false])
|
111
|
+
d.run do
|
112
|
+
d.emit(record)
|
113
|
+
end
|
114
|
+
|
115
|
+
assert_requested(splunk_request)
|
116
|
+
end
|
117
|
+
|
118
|
+
def test_should_send_event_as_log4j_format_when_configured
|
119
|
+
log_time = '2017-07-02 20:52:39'
|
120
|
+
log_time_millis = '1499028759000'
|
121
|
+
log_event = 'data'
|
122
|
+
|
123
|
+
splunk_request = stub_request(:post, SPLUNK_URL)
|
124
|
+
.with(body: hash_including({'time' => log_time_millis, 'event' => "#{log_time} #{log_event}"}))
|
125
|
+
|
126
|
+
d = create_driver_splunkhec(CONFIG + %[usejson false])
|
127
|
+
d.run do
|
128
|
+
d.emit({'time' => log_time, 'message' => log_event})
|
129
|
+
end
|
130
|
+
|
131
|
+
assert_requested(splunk_request)
|
132
|
+
end
|
133
|
+
|
134
|
+
def test_should_send_event_as_json_when_configured
|
135
|
+
record = {'message' => 'data'}
|
136
|
+
|
137
|
+
splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'event' => record}))
|
138
|
+
|
139
|
+
d = create_driver_splunkhec(CONFIG + %[send_event_as_json true])
|
140
|
+
d.run do
|
141
|
+
d.emit(record)
|
142
|
+
end
|
143
|
+
|
144
|
+
assert_requested(splunk_request)
|
145
|
+
end
|
146
|
+
|
147
|
+
def test_should_batch_post_all_events_in_chunk_when_configured
|
148
|
+
record1 = {'message' => 'data'}
|
149
|
+
record2 = {'message' => 'more data'}
|
150
|
+
|
151
|
+
splunk_request = stub_request(:post, SPLUNK_URL).with(body: /\"event\" :#{record1.to_json}.*\"event\" :#{record2.to_json}/m)
|
152
|
+
|
153
|
+
d = create_driver_splunkhec(CONFIG + %[
|
154
|
+
send_event_as_json true
|
155
|
+
send_batched_events true])
|
156
|
+
|
157
|
+
d.run do
|
158
|
+
d.emit(record1)
|
159
|
+
d.emit(record2)
|
160
|
+
end
|
161
|
+
|
162
|
+
assert_requested(splunk_request)
|
26
163
|
end
|
27
164
|
|
28
165
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-splunkhec
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '1.
|
4
|
+
version: '1.2'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Coen Meerbeek
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-08
|
11
|
+
date: 2017-09-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -90,6 +90,20 @@ dependencies:
|
|
90
90
|
- - ">="
|
91
91
|
- !ruby/object:Gem::Version
|
92
92
|
version: 3.1.0
|
93
|
+
- !ruby/object:Gem::Dependency
|
94
|
+
name: webmock
|
95
|
+
requirement: !ruby/object:Gem::Requirement
|
96
|
+
requirements:
|
97
|
+
- - ">="
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: '3.0'
|
100
|
+
type: :development
|
101
|
+
prerelease: false
|
102
|
+
version_requirements: !ruby/object:Gem::Requirement
|
103
|
+
requirements:
|
104
|
+
- - ">="
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: '3.0'
|
93
107
|
description: Output plugin for the Splunk HTTP Event Collector.
|
94
108
|
email: cmeerbeek@gmail.com
|
95
109
|
executables: []
|