fluent-plugin-splunk-http-eventcollector 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +24 -14
- data/README.md +2 -0
- data/fluent-plugin-splunk-http-eventcollector.gemspec +5 -4
- data/lib/fluent/plugin/out_splunk-http-eventcollector.rb +115 -29
- data/test/helper.rb +1 -0
- data/test/plugin/test_out_splunk-http-eventcollector.rb +149 -4
- metadata +32 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f0b634d4b7da2f0a0461d6edaa9f0f785e3de843
|
4
|
+
data.tar.gz: 17fed2d133c051734081f7a5abff20ab2d9dca66
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 00fdd38ca4382e5219334fb82553a15df387d4871b5c0d7425b67de6b67fa40432f4ede9d32a7899953e0122897632b7255a0a69b07d5677c3d413ce48facc4e
|
7
|
+
data.tar.gz: 1a9dea180b260fb78079b590f3f86d296ebac8d452914e4721a76a0fd1a70324c19805a0c5dbeae7d711a8e1f3f79977ea2f1b87174b1477cc7c35d9f7cf0480
|
data/.gitignore
CHANGED
@@ -1,17 +1,27 @@
|
|
1
|
+
### Ruby ###
|
1
2
|
*.gem
|
2
3
|
*.rbc
|
3
|
-
|
4
|
-
|
5
|
-
|
4
|
+
/.config
|
5
|
+
/coverage/
|
6
|
+
/InstalledFiles
|
7
|
+
/pkg/
|
8
|
+
/spec/reports/
|
9
|
+
/spec/examples.txt
|
10
|
+
/test/tmp/
|
11
|
+
/test/version_tmp/
|
12
|
+
/tmp/
|
13
|
+
|
14
|
+
## Documentation cache and generated files:
|
15
|
+
/.yardoc/
|
16
|
+
/_yardoc/
|
17
|
+
/doc/
|
18
|
+
/rdoc/
|
19
|
+
|
20
|
+
## Environment normalization:
|
21
|
+
/.bundle/
|
22
|
+
/vendor/bundle
|
23
|
+
/lib/bundler/man/
|
24
|
+
|
6
25
|
Gemfile.lock
|
7
|
-
|
8
|
-
|
9
|
-
coverage
|
10
|
-
doc/
|
11
|
-
lib/bundler/man
|
12
|
-
pkg
|
13
|
-
rdoc
|
14
|
-
spec/reports
|
15
|
-
test/tmp
|
16
|
-
test/version_tmp
|
17
|
-
tmp
|
26
|
+
.ruby-version
|
27
|
+
.ruby-gemset
|
data/README.md
CHANGED
@@ -5,6 +5,8 @@ Splunk output plugin for Fluent event collector.
|
|
5
5
|
This plugin interfaces with the Splunk HTTP Event Collector:
|
6
6
|
http://dev.splunk.com/view/event-collector/SP-CAAAE6M
|
7
7
|
|
8
|
+
[](https://travis-ci.org/brycied00d/fluent-plugin-splunk-http-eventcollector)
|
9
|
+
|
8
10
|
## Basic Example
|
9
11
|
|
10
12
|
<match **>
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.name = "fluent-plugin-splunk-http-eventcollector"
|
6
|
-
gem.version = "0.
|
6
|
+
gem.version = "0.2.0"
|
7
7
|
gem.authors = ["Bryce Chidester"]
|
8
8
|
gem.email = ["bryce.chidester@calyptix.com"]
|
9
9
|
gem.summary = "Splunk output plugin for Fluentd"
|
@@ -16,12 +16,13 @@ Gem::Specification.new do |gem|
|
|
16
16
|
"fluent-plugin-splunk-http-eventcollector.gemspec",
|
17
17
|
"lib/fluent/plugin/out_splunk-http-eventcollector.rb",
|
18
18
|
"test/plugin/test_out_splunk-http-eventcollector.rb" ]
|
19
|
-
gem.test_files = [ "test/helper.rb",
|
19
|
+
gem.test_files = [ "test/helper.rb",
|
20
20
|
"test/plugin/test_out_splunk-http-eventcollector.rb" ]
|
21
21
|
gem.require_paths = ["lib"]
|
22
22
|
|
23
|
+
gem.add_development_dependency "rake"
|
23
24
|
gem.add_development_dependency "test-unit", '~> 3.1'
|
24
|
-
gem.
|
25
|
+
gem.add_development_dependency "webmock", '~> 2.3', '>= 2.3.2'
|
26
|
+
gem.add_runtime_dependency "fluentd", '~> 0.12.12'
|
25
27
|
gem.add_runtime_dependency "net-http-persistent", '~> 2.9'
|
26
|
-
gem.add_runtime_dependency "fluent-mixin-rewrite-tag-name", '~> 0.1.0'
|
27
28
|
end
|
@@ -25,37 +25,47 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
25
25
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
26
26
|
|
27
27
|
=end
|
28
|
-
|
28
|
+
|
29
|
+
# Splunk HTTP Event collector docs
|
30
|
+
# http://dev.splunk.com/view/event-collector/SP-CAAAE6M
|
31
|
+
# http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
|
29
32
|
|
30
33
|
module Fluent
|
31
34
|
class SplunkHTTPEventcollectorOutput < BufferedOutput
|
32
|
-
|
33
|
-
Plugin.register_output('splunk-http-eventcollector', self)
|
34
35
|
|
35
|
-
|
36
|
-
include Fluent::Mixin::RewriteTagName
|
36
|
+
Plugin.register_output('splunk-http-eventcollector', self)
|
37
37
|
|
38
38
|
config_param :test_mode, :bool, :default => false
|
39
|
-
|
39
|
+
|
40
40
|
config_param :server, :string, :default => 'localhost:8088'
|
41
41
|
config_param :verify, :bool, :default => true
|
42
42
|
config_param :token, :string, :default => nil
|
43
|
-
|
43
|
+
|
44
44
|
# Event parameters
|
45
45
|
config_param :protocol, :string, :default => 'https'
|
46
46
|
config_param :host, :string, :default => nil
|
47
47
|
config_param :index, :string, :default => 'main'
|
48
48
|
config_param :all_items, :bool, :default => false
|
49
|
-
|
49
|
+
|
50
50
|
config_param :sourcetype, :string, :default => 'fluentd'
|
51
51
|
config_param :source, :string, :default => nil
|
52
52
|
config_param :post_retry_max, :integer, :default => 5
|
53
53
|
config_param :post_retry_interval, :integer, :default => 5
|
54
|
-
|
54
|
+
|
55
55
|
# TODO Find better upper limits
|
56
56
|
config_param :batch_size_limit, :integer, :default => 262144 # 65535
|
57
57
|
#config_param :batch_event_limit, :integer, :default => 100
|
58
|
-
|
58
|
+
|
59
|
+
# Whether to allow non-UTF-8 characters in user logs. If set to true, any
|
60
|
+
# non-UTF-8 character would be replaced by the string specified by
|
61
|
+
# 'non_utf8_replacement_string'. If set to false, any non-UTF-8 character
|
62
|
+
# would trigger the plugin to error out.
|
63
|
+
config_param :coerce_to_utf8, :bool, :default => true
|
64
|
+
|
65
|
+
# If 'coerce_to_utf8' is set to true, any non-UTF-8 character would be
|
66
|
+
# replaced by the string specified here.
|
67
|
+
config_param :non_utf8_replacement_string, :string, :default => ' '
|
68
|
+
|
59
69
|
# Called on class load (class initializer)
|
60
70
|
def initialize
|
61
71
|
super
|
@@ -63,7 +73,34 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
63
73
|
require 'net/http/persistent'
|
64
74
|
require 'openssl'
|
65
75
|
end # initialize
|
66
|
-
|
76
|
+
|
77
|
+
# Thanks to
|
78
|
+
# https://github.com/kazegusuri/fluent-plugin-prometheus/blob/348c112d/lib/fluent/plugin/prometheus.rb
|
79
|
+
def self.placeholder_expander(log)
|
80
|
+
# Use internal class in order to expand placeholder
|
81
|
+
if defined?(Fluent::Filter) # for v0.12, built-in PlaceholderExpander
|
82
|
+
begin
|
83
|
+
require 'fluent/plugin/filter_record_transformer'
|
84
|
+
if defined?(Fluent::Plugin::RecordTransformerFilter::PlaceholderExpander)
|
85
|
+
# for v0.14
|
86
|
+
return Fluent::Plugin::RecordTransformerFilter::PlaceholderExpander.new(log: log)
|
87
|
+
else
|
88
|
+
# for v0.12
|
89
|
+
return Fluent::RecordTransformerFilter::PlaceholderExpander.new(log: log)
|
90
|
+
end
|
91
|
+
rescue LoadError => e
|
92
|
+
raise ConfigError, "cannot find filter_record_transformer plugin: #{e.message}"
|
93
|
+
end
|
94
|
+
else # for v0.10, use PlaceholderExapander in fluent-plugin-record-reformer plugin
|
95
|
+
begin
|
96
|
+
require 'fluent/plugin/out_record_reformer.rb'
|
97
|
+
return Fluent::RecordReformerOutput::PlaceholderExpander.new(log: log)
|
98
|
+
rescue LoadError => e
|
99
|
+
raise ConfigError, "cannot find fluent-plugin-record-reformer: #{e.message}"
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
67
104
|
## This method is called before starting.
|
68
105
|
## 'conf' is a Hash that includes configuration parameters.
|
69
106
|
## If the configuration is invalid, raise Fluent::ConfigError.
|
@@ -75,9 +112,12 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
75
112
|
rescue
|
76
113
|
raise ConfigError, "Unable to parse the server into a URI."
|
77
114
|
end
|
115
|
+
|
116
|
+
@placeholder_expander = Fluent::SplunkHTTPEventcollectorOutput.placeholder_expander(log)
|
117
|
+
@hostname = Socket.gethostname
|
78
118
|
# TODO Add other robust input/syntax checks.
|
79
119
|
end # configure
|
80
|
-
|
120
|
+
|
81
121
|
## This method is called when starting.
|
82
122
|
## Open sockets or files here.
|
83
123
|
def start
|
@@ -88,46 +128,55 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
88
128
|
@http.override_headers['Content-Type'] = 'application/json'
|
89
129
|
@http.override_headers['User-Agent'] = 'fluent-plugin-splunk-http-eventcollector/0.0.1'
|
90
130
|
@http.override_headers['Authorization'] = "Splunk #{@token}"
|
91
|
-
|
131
|
+
|
92
132
|
log.trace "initialized for splunk-http-eventcollector"
|
93
133
|
end
|
94
|
-
|
134
|
+
|
95
135
|
## This method is called when shutting down.
|
96
136
|
## Shutdown the thread and close sockets or files here.
|
97
137
|
def shutdown
|
98
138
|
super
|
99
139
|
log.trace "splunk-http-eventcollector(shutdown) called"
|
100
|
-
|
140
|
+
|
101
141
|
@http.shutdown
|
102
142
|
log.trace "shutdown from splunk-http-eventcollector"
|
103
143
|
end # shutdown
|
104
|
-
|
144
|
+
|
105
145
|
## This method is called when an event reaches to Fluentd. (like unbuffered emit())
|
106
146
|
## Convert the event to a raw string.
|
107
147
|
def format(tag, time, record)
|
108
148
|
#log.trace "splunk-http-eventcollector(format) called"
|
109
149
|
# Basic object for Splunk. Note explicit type-casting to avoid accidental errors.
|
110
|
-
|
150
|
+
|
151
|
+
placeholder_values = {
|
152
|
+
'tag' => tag,
|
153
|
+
'tag_parts' => tag.split('.'),
|
154
|
+
'hostname' => @hostname,
|
155
|
+
'record' => record
|
156
|
+
}
|
157
|
+
|
158
|
+
placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
|
111
159
|
|
112
160
|
splunk_object = Hash[
|
113
161
|
"time" => time.to_i,
|
114
|
-
"source" => if @source.nil? then tag.to_s else @placeholder_expander.expand(@source) end,
|
115
|
-
"sourcetype" => @placeholder_expander.expand(@sourcetype.to_s),
|
162
|
+
"source" => if @source.nil? then tag.to_s else @placeholder_expander.expand(@source, placeholders) end,
|
163
|
+
"sourcetype" => @placeholder_expander.expand(@sourcetype.to_s, placeholders),
|
116
164
|
"host" => @host.to_s,
|
117
|
-
"index" => @placeholder_expander.expand(@index)
|
165
|
+
"index" => @placeholder_expander.expand(@index, placeholders)
|
118
166
|
]
|
119
167
|
# TODO: parse different source types as expected: KVP, JSON, TEXT
|
120
168
|
if @all_items
|
121
|
-
|
169
|
+
splunk_object["event"] = convert_to_utf8(record)
|
122
170
|
else
|
123
|
-
|
171
|
+
splunk_object["event"] = convert_to_utf8(record["message"])
|
124
172
|
end
|
173
|
+
|
125
174
|
json_event = splunk_object.to_json
|
126
175
|
#log.debug "Generated JSON(#{json_event.class.to_s}): #{json_event.to_s}"
|
127
176
|
#log.debug "format: returning: #{[tag, record].to_json.to_s}"
|
128
177
|
json_event
|
129
178
|
end
|
130
|
-
|
179
|
+
|
131
180
|
# By this point, fluentd has decided its buffer is full and it's time to flush
|
132
181
|
# it. chunk.read is a concatenated string of JSON.to_s objects. Simply POST
|
133
182
|
# them to Splunk and go about our life.
|
@@ -140,7 +189,7 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
140
189
|
## NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
|
141
190
|
def write(chunk)
|
142
191
|
log.trace "splunk-http-eventcollector(write) called"
|
143
|
-
|
192
|
+
|
144
193
|
# Break the concatenated string of JSON-formatted events into an Array
|
145
194
|
split_chunk = chunk.read.split("}{").each do |x|
|
146
195
|
# Reconstruct the opening{/closing} that #split() strips off.
|
@@ -153,12 +202,12 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
153
202
|
# Don't care about the number of events so much as the POST size (bytes)
|
154
203
|
#if split_chunk.size > @batch_event_limit
|
155
204
|
# log.warn "Fluentd is attempting to push #{numfmt(split_chunk.size)} " +
|
156
|
-
# "events in a single push to Splunk. The configured limit is " +
|
205
|
+
# "events in a single push to Splunk. The configured limit is " +
|
157
206
|
# "#{numfmt(@batch_event_limit)}."
|
158
207
|
#end
|
159
208
|
if chunk.read.bytesize > @batch_size_limit
|
160
209
|
log.warn "Fluentd is attempting to push #{numfmt(chunk.read.bytesize)} " +
|
161
|
-
"bytes in a single push to Splunk. The configured limit is " +
|
210
|
+
"bytes in a single push to Splunk. The configured limit is " +
|
162
211
|
"#{numfmt(@batch_size_limit)} bytes."
|
163
212
|
newbuffer = Array.new
|
164
213
|
split_chunk_counter = 0
|
@@ -188,7 +237,7 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
188
237
|
return push_buffer chunk.read
|
189
238
|
end # if chunk.read.bytesize > @batch_size_limit
|
190
239
|
end # write
|
191
|
-
|
240
|
+
|
192
241
|
def push_buffer(body)
|
193
242
|
post = Net::HTTP::Post.new @splunk_uri.request_uri
|
194
243
|
post.body = body
|
@@ -226,13 +275,50 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
226
275
|
else
|
227
276
|
# other errors. fluentd will retry processing on exception
|
228
277
|
# FIXME: this may duplicate logs when using multiple buffers
|
229
|
-
raise "#{@splunk_uri}: #{response.message}"
|
278
|
+
raise "#{@splunk_uri}: #{response.message}\n#{response.body}"
|
230
279
|
end # If response.code
|
231
280
|
end # 1.upto(@post_retry_max)
|
232
281
|
end # push_buffer
|
233
|
-
|
282
|
+
|
234
283
|
def numfmt(input)
|
235
284
|
input.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\1,').reverse
|
236
285
|
end # numfmt
|
286
|
+
|
287
|
+
# Encode as UTF-8. If 'coerce_to_utf8' is set to true in the config, any
|
288
|
+
# non-UTF-8 character would be replaced by the string specified by
|
289
|
+
# 'non_utf8_replacement_string'. If 'coerce_to_utf8' is set to false, any
|
290
|
+
# non-UTF-8 character would trigger the plugin to error out.
|
291
|
+
# Thanks to
|
292
|
+
# https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud/blob/dbc28575/lib/fluent/plugin/out_google_cloud.rb#L1284
|
293
|
+
def convert_to_utf8(input)
|
294
|
+
if input.is_a?(Hash)
|
295
|
+
record = {}
|
296
|
+
input.each do |key, value|
|
297
|
+
record[convert_to_utf8(key)] = convert_to_utf8(value)
|
298
|
+
end
|
299
|
+
|
300
|
+
return record
|
301
|
+
end
|
302
|
+
return input.map { |value| convert_to_utf8(value) } if input.is_a?(Array)
|
303
|
+
return input unless input.respond_to?(:encode)
|
304
|
+
|
305
|
+
if @coerce_to_utf8
|
306
|
+
input.encode(
|
307
|
+
'utf-8',
|
308
|
+
invalid: :replace,
|
309
|
+
undef: :replace,
|
310
|
+
replace: @non_utf8_replacement_string)
|
311
|
+
else
|
312
|
+
begin
|
313
|
+
input.encode('utf-8')
|
314
|
+
rescue EncodingError
|
315
|
+
@log.error 'Encountered encoding issues potentially due to non ' \
|
316
|
+
'UTF-8 characters. To allow non-UTF-8 characters and ' \
|
317
|
+
'replace them with spaces, please set "coerce_to_utf8" ' \
|
318
|
+
'to true.'
|
319
|
+
raise
|
320
|
+
end
|
321
|
+
end
|
322
|
+
end
|
237
323
|
end # class SplunkHTTPEventcollectorOutput
|
238
324
|
end # module Fluent
|
data/test/helper.rb
CHANGED
@@ -18,17 +18,162 @@ class SplunkHTTPEventcollectorOutputTest < Test::Unit::TestCase
|
|
18
18
|
def test_configure
|
19
19
|
# default
|
20
20
|
d = create_driver
|
21
|
-
assert_equal
|
22
|
-
assert_equal '
|
21
|
+
assert_equal nil, d.instance.source
|
22
|
+
assert_equal 'fluentd', d.instance.sourcetype
|
23
23
|
end
|
24
24
|
|
25
25
|
def test_write
|
26
|
+
stub_request(:post, "https://localhost:8089/services/collector").
|
27
|
+
to_return(body: '{"text":"Success","code":0}')
|
28
|
+
|
29
|
+
d = create_driver
|
30
|
+
|
31
|
+
time = Time.parse("2010-01-02 13:14:15 UTC").to_i
|
32
|
+
d.emit({ "message" => "a message"}, time)
|
33
|
+
|
34
|
+
d.run
|
35
|
+
|
36
|
+
assert_requested :post, "https://localhost:8089/services/collector",
|
37
|
+
headers: {
|
38
|
+
"Authorization" => "Splunk changeme",
|
39
|
+
'Content-Type' => 'application/json',
|
40
|
+
'User-Agent' => 'fluent-plugin-splunk-http-eventcollector/0.0.1'
|
41
|
+
},
|
42
|
+
body: { time: time, source:"test", sourcetype: "fluentd", host: "", index: "main", event: "a message" },
|
43
|
+
times: 1
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_expand
|
47
|
+
stub_request(:post, "https://localhost:8089/services/collector").
|
48
|
+
to_return(body: '{"text":"Success","code":0}')
|
49
|
+
|
50
|
+
d = create_driver(CONFIG + %[
|
51
|
+
source ${record["source"]}
|
52
|
+
sourcetype ${tag_parts[0]}
|
53
|
+
])
|
54
|
+
|
55
|
+
time = Time.parse("2010-01-02 13:14:15 UTC").to_i
|
56
|
+
d.emit({"message" => "a message", "source" => "source-from-record"}, time)
|
57
|
+
|
58
|
+
d.run
|
59
|
+
|
60
|
+
assert_requested :post, "https://localhost:8089/services/collector",
|
61
|
+
headers: {"Authorization" => "Splunk changeme"},
|
62
|
+
body: { time: time, source: "source-from-record", sourcetype: "test", host: "", index: "main", event: "a message" },
|
63
|
+
times: 1
|
64
|
+
end
|
65
|
+
|
66
|
+
def test_4XX_error_retry
|
67
|
+
stub_request(:post, "https://localhost:8089/services/collector").
|
68
|
+
with(headers: {"Authorization" => "Splunk changeme"}).
|
69
|
+
to_return(body: '{"text":"Incorrect data format","code":5,"invalid-event-number":0}', status: 400)
|
70
|
+
|
26
71
|
d = create_driver
|
27
72
|
|
28
73
|
time = Time.parse("2010-01-02 13:14:15 UTC").to_i
|
29
|
-
d.emit({"
|
30
|
-
d.
|
74
|
+
d.emit({ "message" => "1" }, time)
|
75
|
+
d.run
|
76
|
+
|
77
|
+
assert_requested :post, "https://localhost:8089/services/collector",
|
78
|
+
headers: {"Authorization" => "Splunk changeme"},
|
79
|
+
body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "1" },
|
80
|
+
times: 1
|
81
|
+
end
|
82
|
+
|
83
|
+
def test_5XX_error_retry
|
84
|
+
request_count = 0
|
85
|
+
stub_request(:post, "https://localhost:8089/services/collector").
|
86
|
+
with(headers: {"Authorization" => "Splunk changeme"}).
|
87
|
+
to_return do |request|
|
88
|
+
request_count += 1
|
89
|
+
|
90
|
+
if request_count < 5
|
91
|
+
{ body: '{"text":"Internal server error","code":8}', status: 500 }
|
92
|
+
else
|
93
|
+
{ body: '{"text":"Success","code":0}', status: 200 }
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
|
98
|
+
d = create_driver(CONFIG + %[
|
99
|
+
post_retry_max 5
|
100
|
+
post_retry_interval 0.1
|
101
|
+
])
|
102
|
+
|
103
|
+
time = Time.parse("2010-01-02 13:14:15 UTC").to_i
|
104
|
+
d.emit({ "message" => "1" }, time)
|
105
|
+
d.run
|
106
|
+
|
107
|
+
assert_requested :post, "https://localhost:8089/services/collector",
|
108
|
+
headers: {"Authorization" => "Splunk changeme"},
|
109
|
+
body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "1" },
|
110
|
+
times: 5
|
111
|
+
end
|
112
|
+
|
113
|
+
def test_write_splitting
|
114
|
+
stub_request(:post, "https://localhost:8089/services/collector").
|
115
|
+
with(headers: {"Authorization" => "Splunk changeme"}).
|
116
|
+
to_return(body: '{"text":"Incorrect data format","code":5,"invalid-event-number":0}', status: 400)
|
117
|
+
|
118
|
+
# A single msg is ~110 bytes
|
119
|
+
d = create_driver(CONFIG + %[
|
120
|
+
batch_size_limit 250
|
121
|
+
])
|
122
|
+
|
123
|
+
time = Time.parse("2010-01-02 13:14:15 UTC").to_i
|
124
|
+
d.emit({"message" => "a" }, time)
|
125
|
+
d.emit({"message" => "b" }, time)
|
126
|
+
d.emit({"message" => "c" }, time)
|
127
|
+
d.run
|
128
|
+
|
129
|
+
assert_requested :post, "https://localhost:8089/services/collector",
|
130
|
+
headers: {"Authorization" => "Splunk changeme"},
|
131
|
+
body:
|
132
|
+
{ time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "a" }.to_json +
|
133
|
+
{ time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "b" }.to_json,
|
134
|
+
times: 1
|
135
|
+
assert_requested :post, "https://localhost:8089/services/collector",
|
136
|
+
headers: {"Authorization" => "Splunk changeme"},
|
137
|
+
body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "c" }.to_json,
|
138
|
+
times: 1
|
139
|
+
assert_requested :post, "https://localhost:8089/services/collector", times: 2
|
140
|
+
end
|
141
|
+
|
142
|
+
def test_utf8
|
143
|
+
stub_request(:post, "https://localhost:8089/services/collector").
|
144
|
+
with(headers: {"Authorization" => "Splunk changeme"}).
|
145
|
+
to_return(body: '{"text":"Success","code":0}')
|
146
|
+
|
147
|
+
d = create_driver(CONFIG + %[
|
148
|
+
all_items true
|
149
|
+
])
|
150
|
+
|
151
|
+
time = Time.parse("2010-01-02 13:14:15 UTC").to_i
|
152
|
+
d.emit({ "some" => { "nested" => "ü†f-8".force_encoding("BINARY"), "with" => ['ü', '†', 'f-8'].map {|c| c.force_encoding("BINARY") } } }, time)
|
153
|
+
d.run
|
31
154
|
|
155
|
+
assert_requested :post, "https://localhost:8089/services/collector",
|
156
|
+
headers: {"Authorization" => "Splunk changeme"},
|
157
|
+
body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: { some: { nested: " f-8", with: [" "," ","f-8"]}}},
|
158
|
+
times: 1
|
159
|
+
end
|
160
|
+
|
161
|
+
def test_utf8
|
162
|
+
stub_request(:post, "https://localhost:8089/services/collector").
|
163
|
+
with(headers: {"Authorization" => "Splunk changeme"}).
|
164
|
+
to_return(body: '{"text":"Success","code":0}')
|
165
|
+
|
166
|
+
d = create_driver(CONFIG + %[
|
167
|
+
all_items true
|
168
|
+
])
|
169
|
+
|
170
|
+
time = Time.parse("2010-01-02 13:14:15 UTC").to_i
|
171
|
+
d.emit({ "some" => { "nested" => "ü†f-8".force_encoding("BINARY"), "with" => ['ü', '†', 'f-8'].map {|c| c.force_encoding("BINARY") } } }, time)
|
32
172
|
d.run
|
173
|
+
|
174
|
+
assert_requested :post, "https://localhost:8089/services/collector",
|
175
|
+
headers: {"Authorization" => "Splunk changeme"},
|
176
|
+
body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: { some: { nested: " f-8", with: [" "," ","f-8"]}}},
|
177
|
+
times: 1
|
33
178
|
end
|
34
179
|
end
|
metadata
CHANGED
@@ -1,15 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-splunk-http-eventcollector
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bryce Chidester
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-04-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
13
27
|
- !ruby/object:Gem::Dependency
|
14
28
|
name: test-unit
|
15
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -25,47 +39,53 @@ dependencies:
|
|
25
39
|
- !ruby/object:Gem::Version
|
26
40
|
version: '3.1'
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
42
|
+
name: webmock
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
30
44
|
requirements:
|
31
45
|
- - "~>"
|
32
46
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
34
|
-
|
47
|
+
version: '2.3'
|
48
|
+
- - ">="
|
49
|
+
- !ruby/object:Gem::Version
|
50
|
+
version: 2.3.2
|
51
|
+
type: :development
|
35
52
|
prerelease: false
|
36
53
|
version_requirements: !ruby/object:Gem::Requirement
|
37
54
|
requirements:
|
38
55
|
- - "~>"
|
39
56
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
57
|
+
version: '2.3'
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 2.3.2
|
41
61
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
62
|
+
name: fluentd
|
43
63
|
requirement: !ruby/object:Gem::Requirement
|
44
64
|
requirements:
|
45
65
|
- - "~>"
|
46
66
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
67
|
+
version: 0.12.12
|
48
68
|
type: :runtime
|
49
69
|
prerelease: false
|
50
70
|
version_requirements: !ruby/object:Gem::Requirement
|
51
71
|
requirements:
|
52
72
|
- - "~>"
|
53
73
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
74
|
+
version: 0.12.12
|
55
75
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
76
|
+
name: net-http-persistent
|
57
77
|
requirement: !ruby/object:Gem::Requirement
|
58
78
|
requirements:
|
59
79
|
- - "~>"
|
60
80
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
81
|
+
version: '2.9'
|
62
82
|
type: :runtime
|
63
83
|
prerelease: false
|
64
84
|
version_requirements: !ruby/object:Gem::Requirement
|
65
85
|
requirements:
|
66
86
|
- - "~>"
|
67
87
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
88
|
+
version: '2.9'
|
69
89
|
description: Splunk output plugin (HTTP Event Collector) for Fluentd event collector
|
70
90
|
email:
|
71
91
|
- bryce.chidester@calyptix.com
|