fluent-plugin-splunk-http-eventcollector 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/Gemfile +6 -0
- data/LICENSE +24 -0
- data/README.md +196 -0
- data/Rakefile +10 -0
- data/fluent-plugin-splunk-http-eventcollector.gemspec +26 -0
- data/lib/fluent/plugin/out_splunk-http-eventcollector.rb +221 -0
- data/test/helper.rb +28 -0
- data/test/plugin/test_out_splunk-http-eventcollector.rb +34 -0
- metadata +99 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a9bb8b2d4ecc9e67d89c91b7c146b3be7396e12d
|
4
|
+
data.tar.gz: 08df84dc5e71141a4558fe2424798062061283dd
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: e51aabb236cd36f73b031c08982741defdc91abc27add9274671678bfdf2ed7ef06db4ad76feb69108d785a39a731681020ecc08dc49fe106c15054190d76844
|
7
|
+
data.tar.gz: b3196810b72dde78988d46495b9447dd9669851135beb8a5d3912fa1cb26fd179cc07f7fc51fa0a370514f19d61905b9c8b039c7779caf9de380c1152140a9de
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
Copyright (c) 2015, Bryce Chidester (Calyptix Security)
|
2
|
+
All rights reserved.
|
3
|
+
|
4
|
+
Redistribution and use in source and binary forms, with or without
|
5
|
+
modification, are permitted provided that the following conditions are met:
|
6
|
+
|
7
|
+
* Redistributions of source code must retain the above copyright notice, this
|
8
|
+
list of conditions and the following disclaimer.
|
9
|
+
|
10
|
+
* Redistributions in binary form must reproduce the above copyright notice,
|
11
|
+
this list of conditions and the following disclaimer in the documentation
|
12
|
+
and/or other materials provided with the distribution.
|
13
|
+
|
14
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
15
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
16
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
17
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
18
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
19
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
20
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
21
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
22
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
23
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
24
|
+
|
data/README.md
ADDED
@@ -0,0 +1,196 @@
|
|
1
|
+
# Fluent::Plugin::SplunkHTTPEventcollector, a plugin for [Fluentd](http://fluentd.org)
|
2
|
+
|
3
|
+
Splunk output plugin for Fluent event collector.
|
4
|
+
|
5
|
+
This plugin interfaces with the Splunk HTTP Event Collector:
|
6
|
+
http://dev.splunk.com/view/event-collector/SP-CAAAE6M
|
7
|
+
|
8
|
+
## Basic Example
|
9
|
+
|
10
|
+
<match **>
|
11
|
+
type splunk-http-eventcollector
|
12
|
+
server 127.0.0.1:8088
|
13
|
+
verify false
|
14
|
+
token YOUR-TOKEN
|
15
|
+
|
16
|
+
# Convert fluent tags to Splunk sources.
|
17
|
+
# If you set an index, "check_index false" is required.
|
18
|
+
host YOUR-HOSTNAME
|
19
|
+
index SOME-INDEX
|
20
|
+
check_index false
|
21
|
+
source {TAG}
|
22
|
+
sourcetype fluent
|
23
|
+
|
24
|
+
# TIMESTAMP: key1="value1" key2="value2" ...
|
25
|
+
time_format unixtime
|
26
|
+
format kvp
|
27
|
+
|
28
|
+
# Memory buffer with a short flush internal.
|
29
|
+
buffer_type memory
|
30
|
+
buffer_queue_limit 16
|
31
|
+
buffer_chunk_limit 8m
|
32
|
+
flush_interval 2s
|
33
|
+
</match>
|
34
|
+
|
35
|
+
## Installation
|
36
|
+
|
37
|
+
Add this line to your application's Gemfile:
|
38
|
+
|
39
|
+
gem 'fluent-plugin-splunk-http-eventcollector'
|
40
|
+
|
41
|
+
And then execute:
|
42
|
+
|
43
|
+
$ bundle
|
44
|
+
|
45
|
+
Or install it yourself as:
|
46
|
+
|
47
|
+
$ gem install fluent-plugin-splunk-http-eventcollector
|
48
|
+
|
49
|
+
## Configuration
|
50
|
+
|
51
|
+
Put the following lines to your fluent.conf:
|
52
|
+
|
53
|
+
<match **>
|
54
|
+
type splunk-http-eventcollector
|
55
|
+
|
56
|
+
# server: Splunk server host and port
|
57
|
+
# default: localhost:8088
|
58
|
+
server localhost:8088
|
59
|
+
|
60
|
+
# verify: SSL server verification
|
61
|
+
# default: true
|
62
|
+
#verify false
|
63
|
+
|
64
|
+
# token: the token issued
|
65
|
+
token YOUR-TOKEN
|
66
|
+
|
67
|
+
#
|
68
|
+
# Event Parameters
|
69
|
+
#
|
70
|
+
|
71
|
+
# host: 'host' parameter passed to Splunk
|
72
|
+
host YOUR-HOSTNAME
|
73
|
+
|
74
|
+
# index: 'index' parameter passed to Splunk (REST only)
|
75
|
+
# default: <none>
|
76
|
+
#index main
|
77
|
+
|
78
|
+
# check_index: 'check-index' parameter passed to Splunk (REST only)
|
79
|
+
# default: <none>
|
80
|
+
#check_index false
|
81
|
+
|
82
|
+
# host: 'source' parameter passed to Splunk
|
83
|
+
# default: {TAG}
|
84
|
+
#
|
85
|
+
# "{TAG}" will be replaced by fluent tags at runtime
|
86
|
+
source {TAG}
|
87
|
+
|
88
|
+
# sourcetype: 'sourcetype' parameter passed to Splunk
|
89
|
+
# default: fluent
|
90
|
+
sourcetype fluent
|
91
|
+
|
92
|
+
#
|
93
|
+
# Formatting Parameters
|
94
|
+
#
|
95
|
+
|
96
|
+
# time_format: the time format of each event
|
97
|
+
# value: none, unixtime, localtime, or any time format string
|
98
|
+
# default: localtime
|
99
|
+
time_format localtime
|
100
|
+
|
101
|
+
# format: the text format of each event
|
102
|
+
# value: json, kvp, or text
|
103
|
+
# default: json
|
104
|
+
#
|
105
|
+
# input = {"x":1, "y":"xyz", "message":"Hello, world!"}
|
106
|
+
#
|
107
|
+
# 'json' is JSON encoding:
|
108
|
+
# {"x":1,"y":"xyz","message":"Hello, world!"}
|
109
|
+
#
|
110
|
+
# 'kvp' is "key=value" pairs, which is automatically detected as fields by Splunk:
|
111
|
+
# x="1" y="xyz" message="Hello, world!"
|
112
|
+
#
|
113
|
+
# 'text' outputs the value of "message" as is, with "key=value" pairs for others:
|
114
|
+
# [x="1" y="xyz"] Hello, world!
|
115
|
+
format json
|
116
|
+
|
117
|
+
#
|
118
|
+
# Buffering Parameters
|
119
|
+
#
|
120
|
+
|
121
|
+
# Standard parameters for buffering. See documentation for details:
|
122
|
+
# http://docs.fluentd.org/articles/buffer-plugin-overview
|
123
|
+
buffer_type memory
|
124
|
+
buffer_queue_limit 16
|
125
|
+
|
126
|
+
# buffer_chunk_limit: The maxium size of POST data in a single API call.
|
127
|
+
#
|
128
|
+
# This value should be reasonablly small since the current implementation
|
129
|
+
# of out_splunk-http-eventcollector converts a chunk to POST data on memory before API calls.
|
130
|
+
# The default value should be good enough.
|
131
|
+
buffer_chunk_limit 8m
|
132
|
+
|
133
|
+
# flush_interval: The interval of API requests.
|
134
|
+
#
|
135
|
+
# Make sure that this value is sufficiently large to make successive API calls.
|
136
|
+
# Note that a different 'source' creates a different API POST, each of which may
|
137
|
+
# take two or more seconds. If you include "{TAG}" in the source parameter and
|
138
|
+
# this 'match' section recieves many tags, a single flush may take long time.
|
139
|
+
# (Run fluentd with -v to see verbose logs.)
|
140
|
+
flush_interval 60s
|
141
|
+
</match>
|
142
|
+
|
143
|
+
## Example
|
144
|
+
|
145
|
+
# Input from applications
|
146
|
+
<source>
|
147
|
+
type forward
|
148
|
+
</source>
|
149
|
+
|
150
|
+
# Input from log files
|
151
|
+
<source>
|
152
|
+
type tail
|
153
|
+
path /var/log/apache2/ssl_access.log
|
154
|
+
tag ssl_access.log
|
155
|
+
format /(?<message>.*)/
|
156
|
+
pos_file /var/log/td-agent/ssl_access.log.pos
|
157
|
+
</source>
|
158
|
+
|
159
|
+
# fluent logs in text format
|
160
|
+
<match fluent.*>
|
161
|
+
type splunk-http-eventcollector
|
162
|
+
protocol rest
|
163
|
+
server splunk.example.com:8089
|
164
|
+
auth admin:pass
|
165
|
+
sourcetype fluentd
|
166
|
+
format text
|
167
|
+
</match>
|
168
|
+
|
169
|
+
# log files in text format without timestamp
|
170
|
+
<match *.log>
|
171
|
+
type splunk-http-eventcollector
|
172
|
+
protocol rest
|
173
|
+
server splunk.example.com:8089
|
174
|
+
auth admin:pass
|
175
|
+
sourcetype log
|
176
|
+
time_format none
|
177
|
+
format text
|
178
|
+
</match>
|
179
|
+
|
180
|
+
# application logs in kvp format
|
181
|
+
<match app.**>
|
182
|
+
type splunk-http-eventcollector
|
183
|
+
protocol rest
|
184
|
+
server splunk.example.com:8089
|
185
|
+
auth admin:pass
|
186
|
+
sourcetype app
|
187
|
+
format kvp
|
188
|
+
</match>
|
189
|
+
|
190
|
+
## Contributing
|
191
|
+
|
192
|
+
1. Fork it
|
193
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
194
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
195
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
196
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.name = "fluent-plugin-splunk-http-eventcollector"
|
6
|
+
gem.version = "0.0.4"
|
7
|
+
gem.authors = ["Bryce Chidester"]
|
8
|
+
gem.email = ["bryce.chidester@calyptix.com"]
|
9
|
+
gem.summary = "Splunk output plugin for Fluentd"
|
10
|
+
gem.description = "Splunk output plugin (HTTP Event Collector) for Fluentd event collector"
|
11
|
+
gem.homepage = "https://github.com/brycied00d/fluent-plugin-splunk-http-eventcollector"
|
12
|
+
gem.license = 'BSD 2-clause'
|
13
|
+
gem.extra_rdoc_files = [ "LICENSE", "README.md" ]
|
14
|
+
gem.files = [ ".gitignore", "Gemfile", "LICENSE", "README.md",
|
15
|
+
"Rakefile", "test/helper.rb",
|
16
|
+
"fluent-plugin-splunk-http-eventcollector.gemspec",
|
17
|
+
"lib/fluent/plugin/out_splunk-http-eventcollector.rb",
|
18
|
+
"test/plugin/test_out_splunk-http-eventcollector.rb" ]
|
19
|
+
gem.test_files = [ "test/helper.rb",
|
20
|
+
"test/plugin/test_out_splunk-http-eventcollector.rb" ]
|
21
|
+
gem.require_paths = ["lib"]
|
22
|
+
|
23
|
+
gem.add_development_dependency "test-unit", '~> 3.1'
|
24
|
+
gem.add_runtime_dependency "fluentd", '~> 0.12'
|
25
|
+
gem.add_runtime_dependency "net-http-persistent", '~> 2.9'
|
26
|
+
end
|
@@ -0,0 +1,221 @@
|
|
1
|
+
=begin
|
2
|
+
|
3
|
+
Copyright (c) 2015, Bryce Chidester (Calyptix Security)
|
4
|
+
All rights reserved.
|
5
|
+
|
6
|
+
Redistribution and use in source and binary forms, with or without
|
7
|
+
modification, are permitted provided that the following conditions are met:
|
8
|
+
|
9
|
+
* Redistributions of source code must retain the above copyright notice, this
|
10
|
+
list of conditions and the following disclaimer.
|
11
|
+
|
12
|
+
* Redistributions in binary form must reproduce the above copyright notice,
|
13
|
+
this list of conditions and the following disclaimer in the documentation
|
14
|
+
and/or other materials provided with the distribution.
|
15
|
+
|
16
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
17
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
18
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
19
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
20
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
21
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
22
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
23
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
24
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
25
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
26
|
+
|
27
|
+
=end
|
28
|
+
|
29
|
+
module Fluent
|
30
|
+
class SplunkHTTPEventcollectorOutput < BufferedOutput
|
31
|
+
Plugin.register_output('splunk-http-eventcollector', self)
|
32
|
+
|
33
|
+
config_param :test_mode, :bool, :default => false
|
34
|
+
|
35
|
+
config_param :server, :string, :default => 'localhost:8088'
|
36
|
+
config_param :verify, :bool, :default => true
|
37
|
+
config_param :token, :string, :default => nil
|
38
|
+
|
39
|
+
# Event parameters
|
40
|
+
config_param :host, :string, :default => nil
|
41
|
+
config_param :index, :string, :default => 'main'
|
42
|
+
|
43
|
+
config_param :post_retry_max, :integer, :default => 5
|
44
|
+
config_param :post_retry_interval, :integer, :default => 5
|
45
|
+
|
46
|
+
# TODO Find better upper limits
|
47
|
+
config_param :batch_size_limit, :integer, :default => 262144 # 65535
|
48
|
+
#config_param :batch_event_limit, :integer, :default => 100
|
49
|
+
|
50
|
+
# Called on class load (class initializer)
|
51
|
+
def initialize
|
52
|
+
super
|
53
|
+
$log.debug "splunk-http-eventcollector(initialize) called"
|
54
|
+
require 'net/http/persistent'
|
55
|
+
require 'openssl'
|
56
|
+
end # initialize
|
57
|
+
|
58
|
+
## This method is called before starting.
|
59
|
+
## 'conf' is a Hash that includes configuration parameters.
|
60
|
+
## If the configuration is invalid, raise Fluent::ConfigError.
|
61
|
+
def configure(conf)
|
62
|
+
super
|
63
|
+
$log.debug "splunk-http-eventcollector(configure) called"
|
64
|
+
begin
|
65
|
+
@splunk_uri = URI "https://#{@server}/services/collector"
|
66
|
+
rescue
|
67
|
+
raise ConfigError, "Unable to parse the server into a URI."
|
68
|
+
end
|
69
|
+
# TODO Add other robust input/syntax checks.
|
70
|
+
end # configure
|
71
|
+
|
72
|
+
## This method is called when starting.
|
73
|
+
## Open sockets or files here.
|
74
|
+
def start
|
75
|
+
super
|
76
|
+
$log.debug "splunk-http-eventcollector(start) called"
|
77
|
+
@http = Net::HTTP::Persistent.new 'fluent-plugin-splunk-http-eventcollector'
|
78
|
+
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @verify
|
79
|
+
@http.override_headers['Content-Type'] = 'application/json'
|
80
|
+
@http.override_headers['User-Agent'] = 'fluent-plugin-splunk-http-eventcollector/0.0.1'
|
81
|
+
@http.override_headers['Authorization'] = "Splunk #{@token}"
|
82
|
+
|
83
|
+
$log.debug "initialized for splunk-http-eventcollector"
|
84
|
+
end
|
85
|
+
|
86
|
+
## This method is called when shutting down.
|
87
|
+
## Shutdown the thread and close sockets or files here.
|
88
|
+
def shutdown
|
89
|
+
super
|
90
|
+
$log.debug "splunk-http-eventcollector(shutdown) called"
|
91
|
+
|
92
|
+
@http.shutdown
|
93
|
+
$log.debug "shutdown from splunk-http-eventcollector"
|
94
|
+
end # shutdown
|
95
|
+
|
96
|
+
## This method is called when an event reaches to Fluentd. (like unbuffered emit())
|
97
|
+
## Convert the event to a raw string.
|
98
|
+
def format(tag, time, record)
|
99
|
+
#$log.debug "splunk-http-eventcollector(format) called"
|
100
|
+
# Basic object for Splunk. Note explicit type-casting to avoid accidental errors.
|
101
|
+
splunk_object = Hash[
|
102
|
+
"event" => record["message"],
|
103
|
+
"time" => time.to_i,
|
104
|
+
"source" => tag.to_s,
|
105
|
+
"host" => @host.to_s,
|
106
|
+
"index" => @index.to_s
|
107
|
+
]
|
108
|
+
json_event = splunk_object.to_json
|
109
|
+
#$log.debug "Generated JSON(#{json_event.class.to_s}): #{json_event.to_s}"
|
110
|
+
#$log.debug "format: returning: #{[tag, record].to_json.to_s}"
|
111
|
+
json_event
|
112
|
+
end
|
113
|
+
|
114
|
+
# By this point, fluentd has decided its buffer is full and it's time to flush
|
115
|
+
# it. chunk.read is a concatenated string of JSON.to_s objects. Simply POST
|
116
|
+
# them to Splunk and go about our life.
|
117
|
+
## This method is called every flush interval. Write the buffer chunk
|
118
|
+
## to files or databases here.
|
119
|
+
## 'chunk' is a buffer chunk that includes multiple formatted
|
120
|
+
## events. You can use 'data = chunk.read' to get all events and
|
121
|
+
## 'chunk.open {|io| ... }' to get IO objects.
|
122
|
+
##
|
123
|
+
## NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
|
124
|
+
def write(chunk)
|
125
|
+
$log.debug "splunk-http-eventcollector(write) called"
|
126
|
+
|
127
|
+
# Break the concatenated string of JSON-formatted events into an Array
|
128
|
+
split_chunk = chunk.read.split("}{").each do |x|
|
129
|
+
# Reconstruct the opening{/closing} that #split() strips off.
|
130
|
+
x.prepend("{") unless x.start_with?("{")
|
131
|
+
x << "}" unless x.end_with?("}")
|
132
|
+
end
|
133
|
+
$log.debug "Pushing #{numfmt(split_chunk.size)} events (" +
|
134
|
+
"#{numfmt(chunk.read.bytesize)} bytes) to Splunk."
|
135
|
+
# If fluentd is pushing too much data to Splunk at once, split up the payload
|
136
|
+
# Don't care about the number of events so much as the POST size (bytes)
|
137
|
+
#if split_chunk.size > @batch_event_limit
|
138
|
+
# $log.warn "Fluentd is attempting to push #{numfmt(split_chunk.size)} " +
|
139
|
+
# "events in a single push to Splunk. The configured limit is " +
|
140
|
+
# "#{numfmt(@batch_event_limit)}."
|
141
|
+
#end
|
142
|
+
if chunk.read.bytesize > @batch_size_limit
|
143
|
+
$log.warn "Fluentd is attempting to push #{numfmt(chunk.read.bytesize)} " +
|
144
|
+
"bytes in a single push to Splunk. The configured limit is " +
|
145
|
+
"#{numfmt(@batch_size_limit)} bytes."
|
146
|
+
newbuffer = Array.new
|
147
|
+
split_chunk_counter = 0
|
148
|
+
split_chunk.each do |c|
|
149
|
+
split_chunk_counter = split_chunk_counter + 1
|
150
|
+
#$log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
|
151
|
+
# "newbuffer.bytesize=#{numfmt(newbuffer.join.bytesize)} + " +
|
152
|
+
# "c.bytesize=#{numfmt(c.bytesize)} ????"
|
153
|
+
if newbuffer.join.bytesize + c.bytesize < @batch_size_limit
|
154
|
+
#$log.debug "Appended!"
|
155
|
+
newbuffer << c
|
156
|
+
else
|
157
|
+
# Reached the limit - push the current newbuffer.join, and reset
|
158
|
+
#$log.debug "Would exceed limit. Flushing newbuffer and continuing."
|
159
|
+
$log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
|
160
|
+
"newbuffer.bytesize=#{numfmt(newbuffer.join.bytesize)} + " +
|
161
|
+
"c.bytesize=#{numfmt(c.bytesize)} > #{numfmt(@batch_size_limit)}, " +
|
162
|
+
"flushing current buffer to Splunk."
|
163
|
+
push_buffer newbuffer.join
|
164
|
+
newbuffer = Array c
|
165
|
+
end # if/else buffer fits limit
|
166
|
+
end # split_chunk.each
|
167
|
+
# Push anything left over.
|
168
|
+
push_buffer newbuffer.join if newbuffer.size
|
169
|
+
return
|
170
|
+
else
|
171
|
+
return push_buffer chunk.read
|
172
|
+
end # if chunk.read.bytesize > @batch_size_limit
|
173
|
+
end # write
|
174
|
+
|
175
|
+
def push_buffer(body)
|
176
|
+
post = Net::HTTP::Post.new @splunk_uri.request_uri
|
177
|
+
post.body = body
|
178
|
+
$log.debug "POST #{@splunk_uri}"
|
179
|
+
if @test_mode
|
180
|
+
$log.debug "TEST_MODE Payload: #{body}"
|
181
|
+
return
|
182
|
+
end
|
183
|
+
# retry up to :post_retry_max times
|
184
|
+
1.upto(@post_retry_max) do |c|
|
185
|
+
response = @http.request @splunk_uri, post
|
186
|
+
$log.debug "=>(#{c}/#{numfmt(@post_retry_max)}) #{response.code} " +
|
187
|
+
"(#{response.message})"
|
188
|
+
# TODO check the actual server response too (it's JSON)
|
189
|
+
if response.code == "200" # and...
|
190
|
+
# success
|
191
|
+
break
|
192
|
+
# TODO check 40X response within post_retry_max and retry
|
193
|
+
elsif response.code.match(/^50/) and c < @post_retry_max
|
194
|
+
# retry
|
195
|
+
$log.warn "#{@splunk_uri}: Server error #{response.code} (" +
|
196
|
+
"#{response.message}). Retrying in #{@post_retry_interval} " +
|
197
|
+
"seconds.\n#{response.body}"
|
198
|
+
sleep @post_retry_interval
|
199
|
+
next
|
200
|
+
elsif response.code.match(/^40/)
|
201
|
+
# user error
|
202
|
+
$log.error "#{@splunk_uri}: #{response.code} (#{response.message})\n#{response.body}"
|
203
|
+
break
|
204
|
+
elsif c < @post_retry_max
|
205
|
+
# retry
|
206
|
+
$log.debug "#{@splunk_uri}: Retrying..."
|
207
|
+
sleep @post_retry_interval
|
208
|
+
next
|
209
|
+
else
|
210
|
+
# other errors. fluentd will retry processing on exception
|
211
|
+
# FIXME: this may duplicate logs when using multiple buffers
|
212
|
+
raise "#{@splunk_uri}: #{response.message}"
|
213
|
+
end # If response.code
|
214
|
+
end # 1.upto(@post_retry_max)
|
215
|
+
end # push_buffer
|
216
|
+
|
217
|
+
def numfmt(input)
|
218
|
+
input.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\1,').reverse
|
219
|
+
end # numfmt
|
220
|
+
end # class SplunkHTTPEventcollectorOutput
|
221
|
+
end # module Fluent
|
data/test/helper.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
require 'fluent/test'
|
15
|
+
unless ENV.has_key?('VERBOSE')
|
16
|
+
nulllogger = Object.new
|
17
|
+
nulllogger.instance_eval {|obj|
|
18
|
+
def method_missing(method, *args)
|
19
|
+
# pass
|
20
|
+
end
|
21
|
+
}
|
22
|
+
$log = nulllogger
|
23
|
+
end
|
24
|
+
|
25
|
+
require 'fluent/plugin/out_splunk-http-eventcollector'
|
26
|
+
|
27
|
+
class Test::Unit::TestCase
|
28
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class SplunkHTTPEventcollectorOutputTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
end
|
7
|
+
|
8
|
+
CONFIG = %[
|
9
|
+
server localhost:8089
|
10
|
+
verify false
|
11
|
+
token changeme
|
12
|
+
]
|
13
|
+
|
14
|
+
def create_driver(conf=CONFIG, tag='test')
|
15
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::SplunkHTTPEventcollectorOutput, tag).configure(conf)
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_configure
|
19
|
+
# default
|
20
|
+
d = create_driver
|
21
|
+
assert_equal '{TAG}', d.instance.source
|
22
|
+
assert_equal '_json', d.instance.sourcetype
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_write
|
26
|
+
d = create_driver
|
27
|
+
|
28
|
+
time = Time.parse("2010-01-02 13:14:15 UTC").to_i
|
29
|
+
d.emit({"a"=>1}, time)
|
30
|
+
d.emit({"a"=>2}, time)
|
31
|
+
|
32
|
+
d.run
|
33
|
+
end
|
34
|
+
end
|
metadata
ADDED
@@ -0,0 +1,99 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-splunk-http-eventcollector
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.4
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Bryce Chidester
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-10-04 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: test-unit
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '3.1'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '3.1'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: fluentd
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0.12'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0.12'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: net-http-persistent
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '2.9'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '2.9'
|
55
|
+
description: Splunk output plugin (HTTP Event Collector) for Fluentd event collector
|
56
|
+
email:
|
57
|
+
- bryce.chidester@calyptix.com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files:
|
61
|
+
- LICENSE
|
62
|
+
- README.md
|
63
|
+
files:
|
64
|
+
- ".gitignore"
|
65
|
+
- Gemfile
|
66
|
+
- LICENSE
|
67
|
+
- README.md
|
68
|
+
- Rakefile
|
69
|
+
- fluent-plugin-splunk-http-eventcollector.gemspec
|
70
|
+
- lib/fluent/plugin/out_splunk-http-eventcollector.rb
|
71
|
+
- test/helper.rb
|
72
|
+
- test/plugin/test_out_splunk-http-eventcollector.rb
|
73
|
+
homepage: https://github.com/brycied00d/fluent-plugin-splunk-http-eventcollector
|
74
|
+
licenses:
|
75
|
+
- BSD 2-clause
|
76
|
+
metadata: {}
|
77
|
+
post_install_message:
|
78
|
+
rdoc_options: []
|
79
|
+
require_paths:
|
80
|
+
- lib
|
81
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
82
|
+
requirements:
|
83
|
+
- - ">="
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - ">="
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '0'
|
91
|
+
requirements: []
|
92
|
+
rubyforge_project:
|
93
|
+
rubygems_version: 2.4.5.1
|
94
|
+
signing_key:
|
95
|
+
specification_version: 4
|
96
|
+
summary: Splunk output plugin for Fluentd
|
97
|
+
test_files:
|
98
|
+
- test/helper.rb
|
99
|
+
- test/plugin/test_out_splunk-http-eventcollector.rb
|