fluent-plugin-splunk-http-eventcollector 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +8 -0
- data/fluent-plugin-splunk-http-eventcollector.gemspec +3 -2
- data/lib/fluent/plugin/out_splunk-http-eventcollector.rb +46 -29
- metadata +18 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6af449c8480d06db46282037eecc373581957efe
|
4
|
+
data.tar.gz: 6372e889d37f0c8ec98f540e53e354a5673dcf85
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3d647ecdfcfdcf530b494d4306c7c5df8eec958ea53d806de94073213bc7fe59b1fd92323db598d4bd9721a374f679ad2ed292a9724f28a19767e7345e0a36dc
|
7
|
+
data.tar.gz: e11b0d0fbba97638f5614a4dd3aa1e57b7d116af75ebed13a9e54fa9dc95aee19ee5f9d46256230085c1636644ba084c56c5545bf009f2f7c81498189ce13b47
|
data/README.md
CHANGED
@@ -46,6 +46,10 @@ Or install it yourself as:
|
|
46
46
|
|
47
47
|
$ gem install fluent-plugin-splunk-http-eventcollector
|
48
48
|
|
49
|
+
Whatever is appropriate for your environment. Note: If you're using the
|
50
|
+
`td-agent` package, it brings with it its own "embedded" Ruby environment with
|
51
|
+
either `td-agent-gem` or `/opt/td-agent/embedded/bin/gem` depending on platform.
|
52
|
+
|
49
53
|
## Configuration
|
50
54
|
|
51
55
|
Put the following lines to your fluent.conf:
|
@@ -57,6 +61,10 @@ Put the following lines to your fluent.conf:
|
|
57
61
|
# default: localhost:8088
|
58
62
|
server localhost:8088
|
59
63
|
|
64
|
+
# protocol: Connect to Splunk server via 'http' or 'https'
|
65
|
+
# default: https
|
66
|
+
#protocol http
|
67
|
+
|
60
68
|
# verify: SSL server verification
|
61
69
|
# default: true
|
62
70
|
#verify false
|
@@ -3,13 +3,13 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.name = "fluent-plugin-splunk-http-eventcollector"
|
6
|
-
gem.version = "0.0
|
6
|
+
gem.version = "0.1.0"
|
7
7
|
gem.authors = ["Bryce Chidester"]
|
8
8
|
gem.email = ["bryce.chidester@calyptix.com"]
|
9
9
|
gem.summary = "Splunk output plugin for Fluentd"
|
10
10
|
gem.description = "Splunk output plugin (HTTP Event Collector) for Fluentd event collector"
|
11
11
|
gem.homepage = "https://github.com/brycied00d/fluent-plugin-splunk-http-eventcollector"
|
12
|
-
gem.license = 'BSD
|
12
|
+
gem.license = 'BSD-2-Clause'
|
13
13
|
gem.extra_rdoc_files = [ "LICENSE", "README.md" ]
|
14
14
|
gem.files = [ ".gitignore", "Gemfile", "LICENSE", "README.md",
|
15
15
|
"Rakefile", "test/helper.rb",
|
@@ -23,4 +23,5 @@ Gem::Specification.new do |gem|
|
|
23
23
|
gem.add_development_dependency "test-unit", '~> 3.1'
|
24
24
|
gem.add_runtime_dependency "fluentd", '~> 0.12'
|
25
25
|
gem.add_runtime_dependency "net-http-persistent", '~> 2.9'
|
26
|
+
gem.add_runtime_dependency "fluent-mixin-rewrite-tag-name", '~> 0.1.0'
|
26
27
|
end
|
@@ -25,11 +25,16 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
25
25
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
26
26
|
|
27
27
|
=end
|
28
|
+
require 'fluent/mixin/rewrite_tag_name'
|
28
29
|
|
29
30
|
module Fluent
|
30
31
|
class SplunkHTTPEventcollectorOutput < BufferedOutput
|
31
|
-
Plugin.register_output('splunk-http-eventcollector', self)
|
32
32
|
|
33
|
+
Plugin.register_output('splunk-http-eventcollector', self)
|
34
|
+
|
35
|
+
include Fluent::HandleTagNameMixin
|
36
|
+
include Fluent::Mixin::RewriteTagName
|
37
|
+
|
33
38
|
config_param :test_mode, :bool, :default => false
|
34
39
|
|
35
40
|
config_param :server, :string, :default => 'localhost:8088'
|
@@ -37,9 +42,13 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
37
42
|
config_param :token, :string, :default => nil
|
38
43
|
|
39
44
|
# Event parameters
|
45
|
+
config_param :protocol, :string, :default => 'https'
|
40
46
|
config_param :host, :string, :default => nil
|
41
47
|
config_param :index, :string, :default => 'main'
|
48
|
+
config_param :all_items, :bool, :default => false
|
42
49
|
|
50
|
+
config_param :sourcetype, :string, :default => 'fluentd'
|
51
|
+
config_param :source, :string, :default => nil
|
43
52
|
config_param :post_retry_max, :integer, :default => 5
|
44
53
|
config_param :post_retry_interval, :integer, :default => 5
|
45
54
|
|
@@ -50,7 +59,7 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
50
59
|
# Called on class load (class initializer)
|
51
60
|
def initialize
|
52
61
|
super
|
53
|
-
|
62
|
+
log.trace "splunk-http-eventcollector(initialize) called"
|
54
63
|
require 'net/http/persistent'
|
55
64
|
require 'openssl'
|
56
65
|
end # initialize
|
@@ -60,9 +69,9 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
60
69
|
## If the configuration is invalid, raise Fluent::ConfigError.
|
61
70
|
def configure(conf)
|
62
71
|
super
|
63
|
-
|
72
|
+
log.trace "splunk-http-eventcollector(configure) called"
|
64
73
|
begin
|
65
|
-
@splunk_uri = URI "
|
74
|
+
@splunk_uri = URI "#{@protocol}://#{@server}/services/collector"
|
66
75
|
rescue
|
67
76
|
raise ConfigError, "Unable to parse the server into a URI."
|
68
77
|
end
|
@@ -73,41 +82,49 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
73
82
|
## Open sockets or files here.
|
74
83
|
def start
|
75
84
|
super
|
76
|
-
|
85
|
+
log.trace "splunk-http-eventcollector(start) called"
|
77
86
|
@http = Net::HTTP::Persistent.new 'fluent-plugin-splunk-http-eventcollector'
|
78
87
|
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @verify
|
79
88
|
@http.override_headers['Content-Type'] = 'application/json'
|
80
89
|
@http.override_headers['User-Agent'] = 'fluent-plugin-splunk-http-eventcollector/0.0.1'
|
81
90
|
@http.override_headers['Authorization'] = "Splunk #{@token}"
|
82
91
|
|
83
|
-
|
92
|
+
log.trace "initialized for splunk-http-eventcollector"
|
84
93
|
end
|
85
94
|
|
86
95
|
## This method is called when shutting down.
|
87
96
|
## Shutdown the thread and close sockets or files here.
|
88
97
|
def shutdown
|
89
98
|
super
|
90
|
-
|
99
|
+
log.trace "splunk-http-eventcollector(shutdown) called"
|
91
100
|
|
92
101
|
@http.shutdown
|
93
|
-
|
102
|
+
log.trace "shutdown from splunk-http-eventcollector"
|
94
103
|
end # shutdown
|
95
104
|
|
96
105
|
## This method is called when an event reaches to Fluentd. (like unbuffered emit())
|
97
106
|
## Convert the event to a raw string.
|
98
107
|
def format(tag, time, record)
|
99
|
-
|
108
|
+
#log.trace "splunk-http-eventcollector(format) called"
|
100
109
|
# Basic object for Splunk. Note explicit type-casting to avoid accidental errors.
|
110
|
+
@placeholder_expander.set_tag(tag)
|
111
|
+
|
101
112
|
splunk_object = Hash[
|
102
|
-
"event" => record["message"],
|
103
113
|
"time" => time.to_i,
|
104
|
-
"source" => tag.to_s,
|
114
|
+
"source" => if @source.nil? then tag.to_s else @placeholder_expander.expand(@source) end,
|
115
|
+
"sourcetype" => @placeholder_expander.expand(@sourcetype.to_s),
|
105
116
|
"host" => @host.to_s,
|
106
|
-
"index" =>
|
107
|
-
|
117
|
+
"index" => @placeholder_expander.expand(@index)
|
118
|
+
]
|
119
|
+
# TODO: parse different source types as expected: KVP, JSON, TEXT
|
120
|
+
if @all_items
|
121
|
+
splunk_object["event"] = record
|
122
|
+
else
|
123
|
+
splunk_object["event"] = record["message"]
|
124
|
+
end
|
108
125
|
json_event = splunk_object.to_json
|
109
|
-
|
110
|
-
|
126
|
+
#log.debug "Generated JSON(#{json_event.class.to_s}): #{json_event.to_s}"
|
127
|
+
#log.debug "format: returning: #{[tag, record].to_json.to_s}"
|
111
128
|
json_event
|
112
129
|
end
|
113
130
|
|
@@ -122,7 +139,7 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
122
139
|
##
|
123
140
|
## NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
|
124
141
|
def write(chunk)
|
125
|
-
|
142
|
+
log.trace "splunk-http-eventcollector(write) called"
|
126
143
|
|
127
144
|
# Break the concatenated string of JSON-formatted events into an Array
|
128
145
|
split_chunk = chunk.read.split("}{").each do |x|
|
@@ -130,33 +147,33 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
130
147
|
x.prepend("{") unless x.start_with?("{")
|
131
148
|
x << "}" unless x.end_with?("}")
|
132
149
|
end
|
133
|
-
|
150
|
+
log.debug "Pushing #{numfmt(split_chunk.size)} events (" +
|
134
151
|
"#{numfmt(chunk.read.bytesize)} bytes) to Splunk."
|
135
152
|
# If fluentd is pushing too much data to Splunk at once, split up the payload
|
136
153
|
# Don't care about the number of events so much as the POST size (bytes)
|
137
154
|
#if split_chunk.size > @batch_event_limit
|
138
|
-
#
|
155
|
+
# log.warn "Fluentd is attempting to push #{numfmt(split_chunk.size)} " +
|
139
156
|
# "events in a single push to Splunk. The configured limit is " +
|
140
157
|
# "#{numfmt(@batch_event_limit)}."
|
141
158
|
#end
|
142
159
|
if chunk.read.bytesize > @batch_size_limit
|
143
|
-
|
160
|
+
log.warn "Fluentd is attempting to push #{numfmt(chunk.read.bytesize)} " +
|
144
161
|
"bytes in a single push to Splunk. The configured limit is " +
|
145
162
|
"#{numfmt(@batch_size_limit)} bytes."
|
146
163
|
newbuffer = Array.new
|
147
164
|
split_chunk_counter = 0
|
148
165
|
split_chunk.each do |c|
|
149
166
|
split_chunk_counter = split_chunk_counter + 1
|
150
|
-
|
167
|
+
#log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
|
151
168
|
# "newbuffer.bytesize=#{numfmt(newbuffer.join.bytesize)} + " +
|
152
169
|
# "c.bytesize=#{numfmt(c.bytesize)} ????"
|
153
170
|
if newbuffer.join.bytesize + c.bytesize < @batch_size_limit
|
154
|
-
|
171
|
+
#log.debug "Appended!"
|
155
172
|
newbuffer << c
|
156
173
|
else
|
157
174
|
# Reached the limit - push the current newbuffer.join, and reset
|
158
|
-
|
159
|
-
|
175
|
+
#log.debug "Would exceed limit. Flushing newbuffer and continuing."
|
176
|
+
log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
|
160
177
|
"newbuffer.bytesize=#{numfmt(newbuffer.join.bytesize)} + " +
|
161
178
|
"c.bytesize=#{numfmt(c.bytesize)} > #{numfmt(@batch_size_limit)}, " +
|
162
179
|
"flushing current buffer to Splunk."
|
@@ -175,15 +192,15 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
175
192
|
def push_buffer(body)
|
176
193
|
post = Net::HTTP::Post.new @splunk_uri.request_uri
|
177
194
|
post.body = body
|
178
|
-
|
195
|
+
log.debug "POST #{@splunk_uri}"
|
179
196
|
if @test_mode
|
180
|
-
|
197
|
+
log.debug "TEST_MODE Payload: #{body}"
|
181
198
|
return
|
182
199
|
end
|
183
200
|
# retry up to :post_retry_max times
|
184
201
|
1.upto(@post_retry_max) do |c|
|
185
202
|
response = @http.request @splunk_uri, post
|
186
|
-
|
203
|
+
log.debug "=>(#{c}/#{numfmt(@post_retry_max)}) #{response.code} " +
|
187
204
|
"(#{response.message})"
|
188
205
|
# TODO check the actual server response too (it's JSON)
|
189
206
|
if response.code == "200" # and...
|
@@ -192,18 +209,18 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
|
|
192
209
|
# TODO check 40X response within post_retry_max and retry
|
193
210
|
elsif response.code.match(/^50/) and c < @post_retry_max
|
194
211
|
# retry
|
195
|
-
|
212
|
+
log.warn "#{@splunk_uri}: Server error #{response.code} (" +
|
196
213
|
"#{response.message}). Retrying in #{@post_retry_interval} " +
|
197
214
|
"seconds.\n#{response.body}"
|
198
215
|
sleep @post_retry_interval
|
199
216
|
next
|
200
217
|
elsif response.code.match(/^40/)
|
201
218
|
# user error
|
202
|
-
|
219
|
+
log.error "#{@splunk_uri}: #{response.code} (#{response.message})\n#{response.body}"
|
203
220
|
break
|
204
221
|
elsif c < @post_retry_max
|
205
222
|
# retry
|
206
|
-
|
223
|
+
log.debug "#{@splunk_uri}: Retrying..."
|
207
224
|
sleep @post_retry_interval
|
208
225
|
next
|
209
226
|
else
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-splunk-http-eventcollector
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bryce Chidester
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-02-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: test-unit
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '2.9'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: fluent-mixin-rewrite-tag-name
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.1.0
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.1.0
|
55
69
|
description: Splunk output plugin (HTTP Event Collector) for Fluentd event collector
|
56
70
|
email:
|
57
71
|
- bryce.chidester@calyptix.com
|
@@ -72,7 +86,7 @@ files:
|
|
72
86
|
- test/plugin/test_out_splunk-http-eventcollector.rb
|
73
87
|
homepage: https://github.com/brycied00d/fluent-plugin-splunk-http-eventcollector
|
74
88
|
licenses:
|
75
|
-
- BSD
|
89
|
+
- BSD-2-Clause
|
76
90
|
metadata: {}
|
77
91
|
post_install_message:
|
78
92
|
rdoc_options: []
|
@@ -90,7 +104,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
90
104
|
version: '0'
|
91
105
|
requirements: []
|
92
106
|
rubyforge_project:
|
93
|
-
rubygems_version: 2.
|
107
|
+
rubygems_version: 2.6.10
|
94
108
|
signing_key:
|
95
109
|
specification_version: 4
|
96
110
|
summary: Splunk output plugin for Fluentd
|