fluent-plugin-splunk-http-eventcollector-test 0.3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 7ca947bc9db8558af7c0078b36ce1dddced57881
4
+ data.tar.gz: 0e860bbd428a4b4884b5102b5f273b0b67417ffe
5
+ SHA512:
6
+ metadata.gz: e26e4c0c9517980df6de47fa920bf704f0222c81e44fe37abb67baa490b8d5433414ef5dbc1f4081ade5f2fb8fa244f7035fa74c44bd1b6bc34689de9e9aff1b
7
+ data.tar.gz: 73e65d9446f5a197ad25a9350299a3328fd387b2a4d0c7580a2d593ceed6f308e4c6b01abb6ed5e7b5bbe2aa847d8649d98b6453ea790a9b9289ef8f719a47f7
@@ -0,0 +1,27 @@
1
+ ### Ruby ###
2
+ *.gem
3
+ *.rbc
4
+ /.config
5
+ /coverage/
6
+ /InstalledFiles
7
+ /pkg/
8
+ /spec/reports/
9
+ /spec/examples.txt
10
+ /test/tmp/
11
+ /test/version_tmp/
12
+ /tmp/
13
+
14
+ ## Documentation cache and generated files:
15
+ /.yardoc/
16
+ /_yardoc/
17
+ /doc/
18
+ /rdoc/
19
+
20
+ ## Environment normalization:
21
+ /.bundle/
22
+ /vendor/bundle
23
+ /lib/bundler/man/
24
+
25
+ Gemfile.lock
26
+ .ruby-version
27
+ .ruby-gemset
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-splunk-http-eventcollector.gemspec
4
+ gemspec
5
+
6
+ #gem "test-unit"
data/LICENSE ADDED
@@ -0,0 +1,24 @@
1
+ Copyright (c) 2015, Bryce Chidester (Calyptix Security)
2
+ All rights reserved.
3
+
4
+ Redistribution and use in source and binary forms, with or without
5
+ modification, are permitted provided that the following conditions are met:
6
+
7
+ * Redistributions of source code must retain the above copyright notice, this
8
+ list of conditions and the following disclaimer.
9
+
10
+ * Redistributions in binary form must reproduce the above copyright notice,
11
+ this list of conditions and the following disclaimer in the documentation
12
+ and/or other materials provided with the distribution.
13
+
14
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
17
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
18
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
20
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
21
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
22
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24
+
@@ -0,0 +1,206 @@
1
+ # Fluent::Plugin::SplunkHTTPEventcollector, a plugin for [Fluentd](http://fluentd.org)
2
+
3
+ Splunk output plugin for Fluent event collector.
4
+
5
+ This plugin interfaces with the Splunk HTTP Event Collector:
6
+ http://dev.splunk.com/view/event-collector/SP-CAAAE6M
7
+
8
+ [![Build Status](https://travis-ci.org/brycied00d/fluent-plugin-splunk-http-eventcollector.svg?branch=master)](https://travis-ci.org/brycied00d/fluent-plugin-splunk-http-eventcollector)
9
+
10
+ ## Basic Example
11
+
12
+ <match **>
13
+ type splunk-http-eventcollector
14
+ server 127.0.0.1:8088
15
+ verify false
16
+ token YOUR-TOKEN
17
+
18
+ # Convert fluent tags to Splunk sources.
19
+ # If you set an index, "check_index false" is required.
20
+ host YOUR-HOSTNAME
21
+ index SOME-INDEX
22
+ check_index false
23
+ source {TAG}
24
+ sourcetype fluent
25
+
26
+ # TIMESTAMP: key1="value1" key2="value2" ...
27
+ time_format unixtime
28
+ format kvp
29
+
30
+ # Memory buffer with a short flush internal.
31
+ buffer_type memory
32
+ buffer_queue_limit 16
33
+ buffer_chunk_limit 8m
34
+ flush_interval 2s
35
+ </match>
36
+
37
+ ## Installation
38
+
39
+ Add this line to your application's Gemfile:
40
+
41
+ gem 'fluent-plugin-splunk-http-eventcollector'
42
+
43
+ And then execute:
44
+
45
+ $ bundle
46
+
47
+ Or install it yourself as:
48
+
49
+ $ gem install fluent-plugin-splunk-http-eventcollector
50
+
51
+ Whatever is appropriate for your environment. Note: If you're using the
52
+ `td-agent` package, it brings with it its own "embedded" Ruby environment with
53
+ either `td-agent-gem` or `/opt/td-agent/embedded/bin/gem` depending on platform.
54
+
55
+ ## Configuration
56
+
57
+ Put the following lines to your fluent.conf:
58
+
59
+ <match **>
60
+ type splunk-http-eventcollector
61
+
62
+ # server: Splunk server host and port
63
+ # default: localhost:8088
64
+ server localhost:8088
65
+
66
+ # protocol: Connect to Splunk server via 'http' or 'https'
67
+ # default: https
68
+ #protocol http
69
+
70
+ # verify: SSL server verification
71
+ # default: true
72
+ #verify false
73
+
74
+ # token: the token issued
75
+ token YOUR-TOKEN
76
+
77
+ #
78
+ # Event Parameters
79
+ #
80
+
81
+ # host: 'host' parameter passed to Splunk
82
+ host YOUR-HOSTNAME
83
+
84
+ # index: 'index' parameter passed to Splunk (REST only)
85
+ # default: <none>
86
+ #index main
87
+
88
+ # check_index: 'check-index' parameter passed to Splunk (REST only)
89
+ # default: <none>
90
+ #check_index false
91
+
92
+ # host: 'source' parameter passed to Splunk
93
+ # default: {TAG}
94
+ #
95
+ # "{TAG}" will be replaced by fluent tags at runtime
96
+ source {TAG}
97
+
98
+ # sourcetype: 'sourcetype' parameter passed to Splunk
99
+ # default: fluent
100
+ sourcetype fluent
101
+
102
+ #
103
+ # Formatting Parameters
104
+ #
105
+
106
+ # time_format: the time format of each event
107
+ # value: none, unixtime, localtime, or any time format string
108
+ # default: localtime
109
+ time_format localtime
110
+
111
+ # format: the text format of each event
112
+ # value: json, kvp, or text
113
+ # default: json
114
+ #
115
+ # input = {"x":1, "y":"xyz", "message":"Hello, world!"}
116
+ #
117
+ # 'json' is JSON encoding:
118
+ # {"x":1,"y":"xyz","message":"Hello, world!"}
119
+ #
120
+ # 'kvp' is "key=value" pairs, which is automatically detected as fields by Splunk:
121
+ # x="1" y="xyz" message="Hello, world!"
122
+ #
123
+ # 'text' outputs the value of "message" as is, with "key=value" pairs for others:
124
+ # [x="1" y="xyz"] Hello, world!
125
+ format json
126
+
127
+ #
128
+ # Buffering Parameters
129
+ #
130
+
131
+ # Standard parameters for buffering. See documentation for details:
132
+ # http://docs.fluentd.org/articles/buffer-plugin-overview
133
+ buffer_type memory
134
+ buffer_queue_limit 16
135
+
136
+ # buffer_chunk_limit: The maxium size of POST data in a single API call.
137
+ #
138
+ # This value should be reasonablly small since the current implementation
139
+ # of out_splunk-http-eventcollector converts a chunk to POST data on memory before API calls.
140
+ # The default value should be good enough.
141
+ buffer_chunk_limit 8m
142
+
143
+ # flush_interval: The interval of API requests.
144
+ #
145
+ # Make sure that this value is sufficiently large to make successive API calls.
146
+ # Note that a different 'source' creates a different API POST, each of which may
147
+ # take two or more seconds. If you include "{TAG}" in the source parameter and
148
+ # this 'match' section recieves many tags, a single flush may take long time.
149
+ # (Run fluentd with -v to see verbose logs.)
150
+ flush_interval 60s
151
+ </match>
152
+
153
+ ## Example
154
+
155
+ # Input from applications
156
+ <source>
157
+ type forward
158
+ </source>
159
+
160
+ # Input from log files
161
+ <source>
162
+ type tail
163
+ path /var/log/apache2/ssl_access.log
164
+ tag ssl_access.log
165
+ format /(?<message>.*)/
166
+ pos_file /var/log/td-agent/ssl_access.log.pos
167
+ </source>
168
+
169
+ # fluent logs in text format
170
+ <match fluent.*>
171
+ type splunk-http-eventcollector
172
+ protocol rest
173
+ server splunk.example.com:8089
174
+ auth admin:pass
175
+ sourcetype fluentd
176
+ format text
177
+ </match>
178
+
179
+ # log files in text format without timestamp
180
+ <match *.log>
181
+ type splunk-http-eventcollector
182
+ protocol rest
183
+ server splunk.example.com:8089
184
+ auth admin:pass
185
+ sourcetype log
186
+ time_format none
187
+ format text
188
+ </match>
189
+
190
+ # application logs in kvp format
191
+ <match app.**>
192
+ type splunk-http-eventcollector
193
+ protocol rest
194
+ server splunk.example.com:8089
195
+ auth admin:pass
196
+ sourcetype app
197
+ format kvp
198
+ </match>
199
+
200
+ ## Contributing
201
+
202
+ 1. Fork it
203
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
204
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
205
+ 4. Push to the branch (`git push origin my-new-feature`)
206
+ 5. Create new Pull Request
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ require 'rake/testtask'
4
+ Rake::TestTask.new(:test) do |test|
5
+ test.libs << 'lib' << 'test'
6
+ test.pattern = 'test/**/test_*.rb'
7
+ test.verbose = true
8
+ end
9
+
10
+ task :default => :test
@@ -0,0 +1,28 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.name = "fluent-plugin-splunk-http-eventcollector-test"
6
+ gem.version = "0.3.0.1"
7
+ gem.authors = ["Bryce Chidester"]
8
+ gem.email = ["bryce.chidester@calyptix.com"]
9
+ gem.summary = "Splunk output plugin for Fluentd with increased batch_ size_limit"
10
+ gem.description = "Splunk output plugin (HTTP Event Collector) for Fluentd event collector"
11
+ gem.homepage = "https://github.com/brycied00d/fluent-plugin-splunk-http-eventcollector"
12
+ gem.license = 'BSD-2-Clause'
13
+ gem.extra_rdoc_files = [ "LICENSE", "README.md" ]
14
+ gem.files = [ ".gitignore", "Gemfile", "LICENSE", "README.md",
15
+ "Rakefile", "test/helper.rb",
16
+ "fluent-plugin-splunk-http-eventcollector-test.gemspec",
17
+ "lib/fluent/plugin/out_splunk-http-eventcollector.rb",
18
+ "test/plugin/test_out_splunk-http-eventcollector.rb" ]
19
+ gem.test_files = [ "test/helper.rb",
20
+ "test/plugin/test_out_splunk-http-eventcollector.rb" ]
21
+ gem.require_paths = ["lib"]
22
+
23
+ gem.add_development_dependency "rake"
24
+ gem.add_development_dependency "test-unit", '~> 3.1'
25
+ gem.add_development_dependency "webmock", '~> 2.3', '>= 2.3.2'
26
+ gem.add_runtime_dependency "fluentd", '~> 0.12.12'
27
+ gem.add_runtime_dependency "net-http-persistent", '~> 2.9'
28
+ end
@@ -0,0 +1,325 @@
1
+ =begin
2
+
3
+ Copyright (c) 2015, Bryce Chidester (Calyptix Security)
4
+ All rights reserved.
5
+
6
+ Redistribution and use in source and binary forms, with or without
7
+ modification, are permitted provided that the following conditions are met:
8
+
9
+ * Redistributions of source code must retain the above copyright notice, this
10
+ list of conditions and the following disclaimer.
11
+
12
+ * Redistributions in binary form must reproduce the above copyright notice,
13
+ this list of conditions and the following disclaimer in the documentation
14
+ and/or other materials provided with the distribution.
15
+
16
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
20
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
+
27
+ =end
28
+
29
+ # Splunk HTTP Event collector docs
30
+ # http://dev.splunk.com/view/event-collector/SP-CAAAE6M
31
+ # http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
32
+
33
+ module Fluent
34
+ class SplunkHTTPEventcollectorOutput < BufferedOutput
35
+
36
+ Plugin.register_output('splunk-http-eventcollector', self)
37
+
38
+ config_param :test_mode, :bool, :default => false
39
+
40
+ config_param :server, :string, :default => 'localhost:8088'
41
+ config_param :verify, :bool, :default => true
42
+ config_param :token, :string, :default => nil
43
+
44
+ # Event parameters
45
+ config_param :protocol, :string, :default => 'https'
46
+ config_param :host, :string, :default => nil
47
+ config_param :index, :string, :default => 'main'
48
+ config_param :all_items, :bool, :default => false
49
+
50
+ config_param :sourcetype, :string, :default => 'fluentd'
51
+ config_param :source, :string, :default => nil
52
+ config_param :post_retry_max, :integer, :default => 5
53
+ config_param :post_retry_interval, :integer, :default => 5
54
+
55
+ # TODO Find better upper limits
56
+ #config_param :batch_size_limit, :integer, :default => 262144 # 65535
57
+ config_param :batch_size_limit, :integer, :default => 10485760
58
+ #config_param :batch_event_limit, :integer, :default => 100
59
+
60
+ # Whether to allow non-UTF-8 characters in user logs. If set to true, any
61
+ # non-UTF-8 character would be replaced by the string specified by
62
+ # 'non_utf8_replacement_string'. If set to false, any non-UTF-8 character
63
+ # would trigger the plugin to error out.
64
+ config_param :coerce_to_utf8, :bool, :default => true
65
+
66
+ # If 'coerce_to_utf8' is set to true, any non-UTF-8 character would be
67
+ # replaced by the string specified here.
68
+ config_param :non_utf8_replacement_string, :string, :default => ' '
69
+
70
+ # Called on class load (class initializer)
71
+ def initialize
72
+ super
73
+ log.trace "splunk-http-eventcollector(initialize) called"
74
+ require 'net/http/persistent'
75
+ require 'openssl'
76
+ end # initialize
77
+
78
+ # Thanks to
79
+ # https://github.com/kazegusuri/fluent-plugin-prometheus/blob/348c112d/lib/fluent/plugin/prometheus.rb
80
+ def self.placeholder_expander(log)
81
+ # Use internal class in order to expand placeholder
82
+ if defined?(Fluent::Filter) # for v0.12, built-in PlaceholderExpander
83
+ begin
84
+ require 'fluent/plugin/filter_record_transformer'
85
+ if defined?(Fluent::Plugin::RecordTransformerFilter::PlaceholderExpander)
86
+ # for v0.14
87
+ return Fluent::Plugin::RecordTransformerFilter::PlaceholderExpander.new(log: log)
88
+ else
89
+ # for v0.12
90
+ return Fluent::RecordTransformerFilter::PlaceholderExpander.new(log: log)
91
+ end
92
+ rescue LoadError => e
93
+ raise ConfigError, "cannot find filter_record_transformer plugin: #{e.message}"
94
+ end
95
+ else # for v0.10, use PlaceholderExapander in fluent-plugin-record-reformer plugin
96
+ begin
97
+ require 'fluent/plugin/out_record_reformer.rb'
98
+ return Fluent::RecordReformerOutput::PlaceholderExpander.new(log: log)
99
+ rescue LoadError => e
100
+ raise ConfigError, "cannot find fluent-plugin-record-reformer: #{e.message}"
101
+ end
102
+ end
103
+ end
104
+
105
+ ## This method is called before starting.
106
+ ## 'conf' is a Hash that includes configuration parameters.
107
+ ## If the configuration is invalid, raise Fluent::ConfigError.
108
+ def configure(conf)
109
+ super
110
+ log.trace "splunk-http-eventcollector(configure) called"
111
+ begin
112
+ @splunk_uri = URI "#{@protocol}://#{@server}/services/collector"
113
+ rescue
114
+ raise ConfigError, "Unable to parse the server into a URI."
115
+ end
116
+
117
+ @placeholder_expander = Fluent::SplunkHTTPEventcollectorOutput.placeholder_expander(log)
118
+ @hostname = Socket.gethostname
119
+ # TODO Add other robust input/syntax checks.
120
+ end # configure
121
+
122
+ ## This method is called when starting.
123
+ ## Open sockets or files here.
124
+ def start
125
+ super
126
+ log.trace "splunk-http-eventcollector(start) called"
127
+ @http = Net::HTTP::Persistent.new 'fluent-plugin-splunk-http-eventcollector'
128
+ @http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @verify
129
+ @http.override_headers['Content-Type'] = 'application/json'
130
+ @http.override_headers['User-Agent'] = 'fluent-plugin-splunk-http-eventcollector/0.0.1'
131
+ @http.override_headers['Authorization'] = "Splunk #{@token}"
132
+
133
+ log.trace "initialized for splunk-http-eventcollector"
134
+ end
135
+
136
+ ## This method is called when shutting down.
137
+ ## Shutdown the thread and close sockets or files here.
138
+ def shutdown
139
+ super
140
+ log.trace "splunk-http-eventcollector(shutdown) called"
141
+
142
+ @http.shutdown
143
+ log.trace "shutdown from splunk-http-eventcollector"
144
+ end # shutdown
145
+
146
+ ## This method is called when an event reaches to Fluentd. (like unbuffered emit())
147
+ ## Convert the event to a raw string.
148
+ def format(tag, time, record)
149
+ #log.trace "splunk-http-eventcollector(format) called"
150
+ # Basic object for Splunk. Note explicit type-casting to avoid accidental errors.
151
+
152
+ placeholder_values = {
153
+ 'tag' => tag,
154
+ 'tag_parts' => tag.split('.'),
155
+ 'hostname' => @hostname,
156
+ 'record' => record
157
+ }
158
+
159
+ placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
160
+
161
+ splunk_object = Hash[
162
+ "time" => time.to_i,
163
+ "source" => if @source.nil? then tag.to_s else @placeholder_expander.expand(@source, placeholders) end,
164
+ "sourcetype" => @placeholder_expander.expand(@sourcetype.to_s, placeholders),
165
+ "host" => @placeholder_expander.expand(@host.to_s, placeholders),
166
+ "index" => @placeholder_expander.expand(@index, placeholders)
167
+ ]
168
+ # TODO: parse different source types as expected: KVP, JSON, TEXT
169
+ if @all_items
170
+ splunk_object["event"] = convert_to_utf8(record)
171
+ else
172
+ splunk_object["event"] = convert_to_utf8(record["message"])
173
+ end
174
+
175
+ json_event = splunk_object.to_json
176
+ #log.debug "Generated JSON(#{json_event.class.to_s}): #{json_event.to_s}"
177
+ #log.debug "format: returning: #{[tag, record].to_json.to_s}"
178
+ json_event
179
+ end
180
+
181
+ # By this point, fluentd has decided its buffer is full and it's time to flush
182
+ # it. chunk.read is a concatenated string of JSON.to_s objects. Simply POST
183
+ # them to Splunk and go about our life.
184
+ ## This method is called every flush interval. Write the buffer chunk
185
+ ## to files or databases here.
186
+ ## 'chunk' is a buffer chunk that includes multiple formatted
187
+ ## events. You can use 'data = chunk.read' to get all events and
188
+ ## 'chunk.open {|io| ... }' to get IO objects.
189
+ ##
190
+ ## NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
191
+ def write(chunk)
192
+ log.trace "splunk-http-eventcollector(write) called"
193
+
194
+ # Break the concatenated string of JSON-formatted events into an Array
195
+ split_chunk = chunk.read.split("}{").each do |x|
196
+ # Reconstruct the opening{/closing} that #split() strips off.
197
+ x.prepend("{") unless x.start_with?("{")
198
+ x << "}" unless x.end_with?("}")
199
+ end
200
+ log.debug "Pushing #{numfmt(split_chunk.size)} events (" +
201
+ "#{numfmt(chunk.read.bytesize)} bytes) to Splunk."
202
+ # If fluentd is pushing too much data to Splunk at once, split up the payload
203
+ # Don't care about the number of events so much as the POST size (bytes)
204
+ #if split_chunk.size > @batch_event_limit
205
+ # log.warn "Fluentd is attempting to push #{numfmt(split_chunk.size)} " +
206
+ # "events in a single push to Splunk. The configured limit is " +
207
+ # "#{numfmt(@batch_event_limit)}."
208
+ #end
209
+ if chunk.read.bytesize > @batch_size_limit
210
+ log.warn "Fluentd is attempting to push #{numfmt(chunk.read.bytesize)} " +
211
+ "bytes in a single push to Splunk. The configured limit is " +
212
+ "#{numfmt(@batch_size_limit)} bytes."
213
+ newbuffer = Array.new
214
+ split_chunk_counter = 0
215
+ split_chunk.each do |c|
216
+ split_chunk_counter = split_chunk_counter + 1
217
+ #log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
218
+ # "newbuffer.bytesize=#{numfmt(newbuffer.join.bytesize)} + " +
219
+ # "c.bytesize=#{numfmt(c.bytesize)} ????"
220
+ if newbuffer.join.bytesize + c.bytesize < @batch_size_limit
221
+ #log.debug "Appended!"
222
+ newbuffer << c
223
+ else
224
+ # Reached the limit - push the current newbuffer.join, and reset
225
+ #log.debug "Would exceed limit. Flushing newbuffer and continuing."
226
+ log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
227
+ "newbuffer.bytesize=#{numfmt(newbuffer.join.bytesize)} + " +
228
+ "c.bytesize=#{numfmt(c.bytesize)} > #{numfmt(@batch_size_limit)}, " +
229
+ "flushing current buffer to Splunk."
230
+ push_buffer newbuffer.join
231
+ newbuffer = Array c
232
+ end # if/else buffer fits limit
233
+ end # split_chunk.each
234
+ # Push anything left over.
235
+ push_buffer newbuffer.join if newbuffer.size
236
+ return
237
+ else
238
+ return push_buffer chunk.read
239
+ end # if chunk.read.bytesize > @batch_size_limit
240
+ end # write
241
+
242
+ def push_buffer(body)
243
+ post = Net::HTTP::Post.new @splunk_uri.request_uri
244
+ post.body = body
245
+ log.debug "POST #{@splunk_uri}"
246
+ if @test_mode
247
+ log.debug "TEST_MODE Payload: #{body}"
248
+ return
249
+ end
250
+ # retry up to :post_retry_max times
251
+ 1.upto(@post_retry_max) do |c|
252
+ response = @http.request @splunk_uri, post
253
+ log.debug "=>(#{c}/#{numfmt(@post_retry_max)}) #{response.code} " +
254
+ "(#{response.message})"
255
+ # TODO check the actual server response too (it's JSON)
256
+ if response.code == "200" # and...
257
+ # success
258
+ break
259
+ # TODO check 40X response within post_retry_max and retry
260
+ elsif response.code.match(/^50/) and c < @post_retry_max
261
+ # retry
262
+ log.warn "#{@splunk_uri}: Server error #{response.code} (" +
263
+ "#{response.message}). Retrying in #{@post_retry_interval} " +
264
+ "seconds.\n#{response.body}"
265
+ sleep @post_retry_interval
266
+ next
267
+ elsif response.code.match(/^40/)
268
+ # user error
269
+ log.error "#{@splunk_uri}: #{response.code} (#{response.message})\n#{response.body}"
270
+ break
271
+ elsif c < @post_retry_max
272
+ # retry
273
+ log.debug "#{@splunk_uri}: Retrying..."
274
+ sleep @post_retry_interval
275
+ next
276
+ else
277
+ # other errors. fluentd will retry processing on exception
278
+ # FIXME: this may duplicate logs when using multiple buffers
279
+ raise "#{@splunk_uri}: #{response.message}\n#{response.body}"
280
+ end # If response.code
281
+ end # 1.upto(@post_retry_max)
282
+ end # push_buffer
283
+
284
+ def numfmt(input)
285
+ input.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\1,').reverse
286
+ end # numfmt
287
+
288
+ # Encode as UTF-8. If 'coerce_to_utf8' is set to true in the config, any
289
+ # non-UTF-8 character would be replaced by the string specified by
290
+ # 'non_utf8_replacement_string'. If 'coerce_to_utf8' is set to false, any
291
+ # non-UTF-8 character would trigger the plugin to error out.
292
+ # Thanks to
293
+ # https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud/blob/dbc28575/lib/fluent/plugin/out_google_cloud.rb#L1284
294
+ def convert_to_utf8(input)
295
+ if input.is_a?(Hash)
296
+ record = {}
297
+ input.each do |key, value|
298
+ record[convert_to_utf8(key)] = convert_to_utf8(value)
299
+ end
300
+
301
+ return record
302
+ end
303
+ return input.map { |value| convert_to_utf8(value) } if input.is_a?(Array)
304
+ return input unless input.respond_to?(:encode)
305
+
306
+ if @coerce_to_utf8
307
+ input.encode(
308
+ 'utf-8',
309
+ invalid: :replace,
310
+ undef: :replace,
311
+ replace: @non_utf8_replacement_string)
312
+ else
313
+ begin
314
+ input.encode('utf-8')
315
+ rescue EncodingError
316
+ @log.error 'Encountered encoding issues potentially due to non ' \
317
+ 'UTF-8 characters. To allow non-UTF-8 characters and ' \
318
+ 'replace them with spaces, please set "coerce_to_utf8" ' \
319
+ 'to true.'
320
+ raise
321
+ end
322
+ end
323
+ end
324
+ end # class SplunkHTTPEventcollectorOutput
325
+ end # module Fluent
@@ -0,0 +1,29 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'fluent/test'
15
+ unless ENV.has_key?('VERBOSE')
16
+ nulllogger = Object.new
17
+ nulllogger.instance_eval {|obj|
18
+ def method_missing(method, *args)
19
+ # pass
20
+ end
21
+ }
22
+ $log = nulllogger
23
+ end
24
+
25
+ require 'webmock/test_unit'
26
+ require 'fluent/plugin/out_splunk-http-eventcollector'
27
+
28
+ class Test::Unit::TestCase
29
+ end
@@ -0,0 +1,179 @@
1
+ require 'helper'
2
+
3
+ class SplunkHTTPEventcollectorOutputTest < Test::Unit::TestCase
4
+ def setup
5
+ Fluent::Test.setup
6
+ end
7
+
8
+ CONFIG = %[
9
+ server localhost:8089
10
+ verify false
11
+ token changeme
12
+ ]
13
+
14
+ def create_driver(conf=CONFIG, tag='test')
15
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::SplunkHTTPEventcollectorOutput, tag).configure(conf)
16
+ end
17
+
18
+ def test_configure
19
+ # default
20
+ d = create_driver
21
+ assert_equal nil, d.instance.source
22
+ assert_equal 'fluentd', d.instance.sourcetype
23
+ end
24
+
25
+ def test_write
26
+ stub_request(:post, "https://localhost:8089/services/collector").
27
+ to_return(body: '{"text":"Success","code":0}')
28
+
29
+ d = create_driver
30
+
31
+ time = Time.parse("2010-01-02 13:14:15 UTC").to_i
32
+ d.emit({ "message" => "a message"}, time)
33
+
34
+ d.run
35
+
36
+ assert_requested :post, "https://localhost:8089/services/collector",
37
+ headers: {
38
+ "Authorization" => "Splunk changeme",
39
+ 'Content-Type' => 'application/json',
40
+ 'User-Agent' => 'fluent-plugin-splunk-http-eventcollector/0.0.1'
41
+ },
42
+ body: { time: time, source:"test", sourcetype: "fluentd", host: "", index: "main", event: "a message" },
43
+ times: 1
44
+ end
45
+
46
+ def test_expand
47
+ stub_request(:post, "https://localhost:8089/services/collector").
48
+ to_return(body: '{"text":"Success","code":0}')
49
+
50
+ d = create_driver(CONFIG + %[
51
+ source ${record["source"]}
52
+ sourcetype ${tag_parts[0]}
53
+ ])
54
+
55
+ time = Time.parse("2010-01-02 13:14:15 UTC").to_i
56
+ d.emit({"message" => "a message", "source" => "source-from-record"}, time)
57
+
58
+ d.run
59
+
60
+ assert_requested :post, "https://localhost:8089/services/collector",
61
+ headers: {"Authorization" => "Splunk changeme"},
62
+ body: { time: time, source: "source-from-record", sourcetype: "test", host: "", index: "main", event: "a message" },
63
+ times: 1
64
+ end
65
+
66
+ def test_4XX_error_retry
67
+ stub_request(:post, "https://localhost:8089/services/collector").
68
+ with(headers: {"Authorization" => "Splunk changeme"}).
69
+ to_return(body: '{"text":"Incorrect data format","code":5,"invalid-event-number":0}', status: 400)
70
+
71
+ d = create_driver
72
+
73
+ time = Time.parse("2010-01-02 13:14:15 UTC").to_i
74
+ d.emit({ "message" => "1" }, time)
75
+ d.run
76
+
77
+ assert_requested :post, "https://localhost:8089/services/collector",
78
+ headers: {"Authorization" => "Splunk changeme"},
79
+ body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "1" },
80
+ times: 1
81
+ end
82
+
83
+ def test_5XX_error_retry
84
+ request_count = 0
85
+ stub_request(:post, "https://localhost:8089/services/collector").
86
+ with(headers: {"Authorization" => "Splunk changeme"}).
87
+ to_return do |request|
88
+ request_count += 1
89
+
90
+ if request_count < 5
91
+ { body: '{"text":"Internal server error","code":8}', status: 500 }
92
+ else
93
+ { body: '{"text":"Success","code":0}', status: 200 }
94
+ end
95
+ end
96
+
97
+
98
+ d = create_driver(CONFIG + %[
99
+ post_retry_max 5
100
+ post_retry_interval 0.1
101
+ ])
102
+
103
+ time = Time.parse("2010-01-02 13:14:15 UTC").to_i
104
+ d.emit({ "message" => "1" }, time)
105
+ d.run
106
+
107
+ assert_requested :post, "https://localhost:8089/services/collector",
108
+ headers: {"Authorization" => "Splunk changeme"},
109
+ body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "1" },
110
+ times: 5
111
+ end
112
+
113
+ def test_write_splitting
114
+ stub_request(:post, "https://localhost:8089/services/collector").
115
+ with(headers: {"Authorization" => "Splunk changeme"}).
116
+ to_return(body: '{"text":"Incorrect data format","code":5,"invalid-event-number":0}', status: 400)
117
+
118
+ # A single msg is ~110 bytes
119
+ d = create_driver(CONFIG + %[
120
+ batch_size_limit 250
121
+ ])
122
+
123
+ time = Time.parse("2010-01-02 13:14:15 UTC").to_i
124
+ d.emit({"message" => "a" }, time)
125
+ d.emit({"message" => "b" }, time)
126
+ d.emit({"message" => "c" }, time)
127
+ d.run
128
+
129
+ assert_requested :post, "https://localhost:8089/services/collector",
130
+ headers: {"Authorization" => "Splunk changeme"},
131
+ body:
132
+ { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "a" }.to_json +
133
+ { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "b" }.to_json,
134
+ times: 1
135
+ assert_requested :post, "https://localhost:8089/services/collector",
136
+ headers: {"Authorization" => "Splunk changeme"},
137
+ body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: "c" }.to_json,
138
+ times: 1
139
+ assert_requested :post, "https://localhost:8089/services/collector", times: 2
140
+ end
141
+
142
+ def test_utf8
143
+ stub_request(:post, "https://localhost:8089/services/collector").
144
+ with(headers: {"Authorization" => "Splunk changeme"}).
145
+ to_return(body: '{"text":"Success","code":0}')
146
+
147
+ d = create_driver(CONFIG + %[
148
+ all_items true
149
+ ])
150
+
151
+ time = Time.parse("2010-01-02 13:14:15 UTC").to_i
152
+ d.emit({ "some" => { "nested" => "ü†f-8".force_encoding("BINARY"), "with" => ['ü', '†', 'f-8'].map {|c| c.force_encoding("BINARY") } } }, time)
153
+ d.run
154
+
155
+ assert_requested :post, "https://localhost:8089/services/collector",
156
+ headers: {"Authorization" => "Splunk changeme"},
157
+ body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: { some: { nested: " f-8", with: [" "," ","f-8"]}}},
158
+ times: 1
159
+ end
160
+
161
+ def test_utf8
162
+ stub_request(:post, "https://localhost:8089/services/collector").
163
+ with(headers: {"Authorization" => "Splunk changeme"}).
164
+ to_return(body: '{"text":"Success","code":0}')
165
+
166
+ d = create_driver(CONFIG + %[
167
+ all_items true
168
+ ])
169
+
170
+ time = Time.parse("2010-01-02 13:14:15 UTC").to_i
171
+ d.emit({ "some" => { "nested" => "ü†f-8".force_encoding("BINARY"), "with" => ['ü', '†', 'f-8'].map {|c| c.force_encoding("BINARY") } } }, time)
172
+ d.run
173
+
174
+ assert_requested :post, "https://localhost:8089/services/collector",
175
+ headers: {"Authorization" => "Splunk changeme"},
176
+ body: { time: time, source: "test", sourcetype: "fluentd", host: "", index: "main", event: { some: { nested: " f-8", with: [" "," ","f-8"]}}},
177
+ times: 1
178
+ end
179
+ end
metadata ADDED
@@ -0,0 +1,133 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-splunk-http-eventcollector-test
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.3.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Bryce Chidester
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-12-19 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rake
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '>='
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - '>='
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: test-unit
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: '3.1'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ~>
39
+ - !ruby/object:Gem::Version
40
+ version: '3.1'
41
+ - !ruby/object:Gem::Dependency
42
+ name: webmock
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ~>
46
+ - !ruby/object:Gem::Version
47
+ version: '2.3'
48
+ - - '>='
49
+ - !ruby/object:Gem::Version
50
+ version: 2.3.2
51
+ type: :development
52
+ prerelease: false
53
+ version_requirements: !ruby/object:Gem::Requirement
54
+ requirements:
55
+ - - ~>
56
+ - !ruby/object:Gem::Version
57
+ version: '2.3'
58
+ - - '>='
59
+ - !ruby/object:Gem::Version
60
+ version: 2.3.2
61
+ - !ruby/object:Gem::Dependency
62
+ name: fluentd
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - ~>
66
+ - !ruby/object:Gem::Version
67
+ version: 0.12.12
68
+ type: :runtime
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ~>
73
+ - !ruby/object:Gem::Version
74
+ version: 0.12.12
75
+ - !ruby/object:Gem::Dependency
76
+ name: net-http-persistent
77
+ requirement: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - ~>
80
+ - !ruby/object:Gem::Version
81
+ version: '2.9'
82
+ type: :runtime
83
+ prerelease: false
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ~>
87
+ - !ruby/object:Gem::Version
88
+ version: '2.9'
89
+ description: Splunk output plugin (HTTP Event Collector) for Fluentd event collector
90
+ email:
91
+ - bryce.chidester@calyptix.com
92
+ executables: []
93
+ extensions: []
94
+ extra_rdoc_files:
95
+ - LICENSE
96
+ - README.md
97
+ files:
98
+ - .gitignore
99
+ - Gemfile
100
+ - LICENSE
101
+ - README.md
102
+ - Rakefile
103
+ - test/helper.rb
104
+ - fluent-plugin-splunk-http-eventcollector-test.gemspec
105
+ - lib/fluent/plugin/out_splunk-http-eventcollector.rb
106
+ - test/plugin/test_out_splunk-http-eventcollector.rb
107
+ homepage: https://github.com/brycied00d/fluent-plugin-splunk-http-eventcollector
108
+ licenses:
109
+ - BSD-2-Clause
110
+ metadata: {}
111
+ post_install_message:
112
+ rdoc_options: []
113
+ require_paths:
114
+ - lib
115
+ required_ruby_version: !ruby/object:Gem::Requirement
116
+ requirements:
117
+ - - '>='
118
+ - !ruby/object:Gem::Version
119
+ version: '0'
120
+ required_rubygems_version: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - '>='
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ requirements: []
126
+ rubyforge_project:
127
+ rubygems_version: 2.0.14.1
128
+ signing_key:
129
+ specification_version: 4
130
+ summary: Splunk output plugin for Fluentd with increased batch_ size_limit
131
+ test_files:
132
+ - test/helper.rb
133
+ - test/plugin/test_out_splunk-http-eventcollector.rb