fluent-plugin-splunk-http-eventcollector 0.0.4 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a9bb8b2d4ecc9e67d89c91b7c146b3be7396e12d
4
- data.tar.gz: 08df84dc5e71141a4558fe2424798062061283dd
3
+ metadata.gz: 6af449c8480d06db46282037eecc373581957efe
4
+ data.tar.gz: 6372e889d37f0c8ec98f540e53e354a5673dcf85
5
5
  SHA512:
6
- metadata.gz: e51aabb236cd36f73b031c08982741defdc91abc27add9274671678bfdf2ed7ef06db4ad76feb69108d785a39a731681020ecc08dc49fe106c15054190d76844
7
- data.tar.gz: b3196810b72dde78988d46495b9447dd9669851135beb8a5d3912fa1cb26fd179cc07f7fc51fa0a370514f19d61905b9c8b039c7779caf9de380c1152140a9de
6
+ metadata.gz: 3d647ecdfcfdcf530b494d4306c7c5df8eec958ea53d806de94073213bc7fe59b1fd92323db598d4bd9721a374f679ad2ed292a9724f28a19767e7345e0a36dc
7
+ data.tar.gz: e11b0d0fbba97638f5614a4dd3aa1e57b7d116af75ebed13a9e54fa9dc95aee19ee5f9d46256230085c1636644ba084c56c5545bf009f2f7c81498189ce13b47
data/README.md CHANGED
@@ -46,6 +46,10 @@ Or install it yourself as:
46
46
 
47
47
  $ gem install fluent-plugin-splunk-http-eventcollector
48
48
 
49
+ Whatever is appropriate for your environment. Note: If you're using the
50
+ `td-agent` package, it brings with it its own "embedded" Ruby environment with
51
+ either `td-agent-gem` or `/opt/td-agent/embedded/bin/gem` depending on platform.
52
+
49
53
  ## Configuration
50
54
 
51
55
  Put the following lines to your fluent.conf:
@@ -57,6 +61,10 @@ Put the following lines to your fluent.conf:
57
61
  # default: localhost:8088
58
62
  server localhost:8088
59
63
 
64
+ # protocol: Connect to Splunk server via 'http' or 'https'
65
+ # default: https
66
+ #protocol http
67
+
60
68
  # verify: SSL server verification
61
69
  # default: true
62
70
  #verify false
@@ -3,13 +3,13 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |gem|
5
5
  gem.name = "fluent-plugin-splunk-http-eventcollector"
6
- gem.version = "0.0.4"
6
+ gem.version = "0.1.0"
7
7
  gem.authors = ["Bryce Chidester"]
8
8
  gem.email = ["bryce.chidester@calyptix.com"]
9
9
  gem.summary = "Splunk output plugin for Fluentd"
10
10
  gem.description = "Splunk output plugin (HTTP Event Collector) for Fluentd event collector"
11
11
  gem.homepage = "https://github.com/brycied00d/fluent-plugin-splunk-http-eventcollector"
12
- gem.license = 'BSD 2-clause'
12
+ gem.license = 'BSD-2-Clause'
13
13
  gem.extra_rdoc_files = [ "LICENSE", "README.md" ]
14
14
  gem.files = [ ".gitignore", "Gemfile", "LICENSE", "README.md",
15
15
  "Rakefile", "test/helper.rb",
@@ -23,4 +23,5 @@ Gem::Specification.new do |gem|
23
23
  gem.add_development_dependency "test-unit", '~> 3.1'
24
24
  gem.add_runtime_dependency "fluentd", '~> 0.12'
25
25
  gem.add_runtime_dependency "net-http-persistent", '~> 2.9'
26
+ gem.add_runtime_dependency "fluent-mixin-rewrite-tag-name", '~> 0.1.0'
26
27
  end
@@ -25,11 +25,16 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25
25
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
26
 
27
27
  =end
28
+ require 'fluent/mixin/rewrite_tag_name'
28
29
 
29
30
  module Fluent
30
31
  class SplunkHTTPEventcollectorOutput < BufferedOutput
31
- Plugin.register_output('splunk-http-eventcollector', self)
32
32
 
33
+ Plugin.register_output('splunk-http-eventcollector', self)
34
+
35
+ include Fluent::HandleTagNameMixin
36
+ include Fluent::Mixin::RewriteTagName
37
+
33
38
  config_param :test_mode, :bool, :default => false
34
39
 
35
40
  config_param :server, :string, :default => 'localhost:8088'
@@ -37,9 +42,13 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
37
42
  config_param :token, :string, :default => nil
38
43
 
39
44
  # Event parameters
45
+ config_param :protocol, :string, :default => 'https'
40
46
  config_param :host, :string, :default => nil
41
47
  config_param :index, :string, :default => 'main'
48
+ config_param :all_items, :bool, :default => false
42
49
 
50
+ config_param :sourcetype, :string, :default => 'fluentd'
51
+ config_param :source, :string, :default => nil
43
52
  config_param :post_retry_max, :integer, :default => 5
44
53
  config_param :post_retry_interval, :integer, :default => 5
45
54
 
@@ -50,7 +59,7 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
50
59
  # Called on class load (class initializer)
51
60
  def initialize
52
61
  super
53
- $log.debug "splunk-http-eventcollector(initialize) called"
62
+ log.trace "splunk-http-eventcollector(initialize) called"
54
63
  require 'net/http/persistent'
55
64
  require 'openssl'
56
65
  end # initialize
@@ -60,9 +69,9 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
60
69
  ## If the configuration is invalid, raise Fluent::ConfigError.
61
70
  def configure(conf)
62
71
  super
63
- $log.debug "splunk-http-eventcollector(configure) called"
72
+ log.trace "splunk-http-eventcollector(configure) called"
64
73
  begin
65
- @splunk_uri = URI "https://#{@server}/services/collector"
74
+ @splunk_uri = URI "#{@protocol}://#{@server}/services/collector"
66
75
  rescue
67
76
  raise ConfigError, "Unable to parse the server into a URI."
68
77
  end
@@ -73,41 +82,49 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
73
82
  ## Open sockets or files here.
74
83
  def start
75
84
  super
76
- $log.debug "splunk-http-eventcollector(start) called"
85
+ log.trace "splunk-http-eventcollector(start) called"
77
86
  @http = Net::HTTP::Persistent.new 'fluent-plugin-splunk-http-eventcollector'
78
87
  @http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @verify
79
88
  @http.override_headers['Content-Type'] = 'application/json'
80
89
  @http.override_headers['User-Agent'] = 'fluent-plugin-splunk-http-eventcollector/0.0.1'
81
90
  @http.override_headers['Authorization'] = "Splunk #{@token}"
82
91
 
83
- $log.debug "initialized for splunk-http-eventcollector"
92
+ log.trace "initialized for splunk-http-eventcollector"
84
93
  end
85
94
 
86
95
  ## This method is called when shutting down.
87
96
  ## Shutdown the thread and close sockets or files here.
88
97
  def shutdown
89
98
  super
90
- $log.debug "splunk-http-eventcollector(shutdown) called"
99
+ log.trace "splunk-http-eventcollector(shutdown) called"
91
100
 
92
101
  @http.shutdown
93
- $log.debug "shutdown from splunk-http-eventcollector"
102
+ log.trace "shutdown from splunk-http-eventcollector"
94
103
  end # shutdown
95
104
 
96
105
  ## This method is called when an event reaches to Fluentd. (like unbuffered emit())
97
106
  ## Convert the event to a raw string.
98
107
  def format(tag, time, record)
99
- #$log.debug "splunk-http-eventcollector(format) called"
108
+ #log.trace "splunk-http-eventcollector(format) called"
100
109
  # Basic object for Splunk. Note explicit type-casting to avoid accidental errors.
110
+ @placeholder_expander.set_tag(tag)
111
+
101
112
  splunk_object = Hash[
102
- "event" => record["message"],
103
113
  "time" => time.to_i,
104
- "source" => tag.to_s,
114
+ "source" => if @source.nil? then tag.to_s else @placeholder_expander.expand(@source) end,
115
+ "sourcetype" => @placeholder_expander.expand(@sourcetype.to_s),
105
116
  "host" => @host.to_s,
106
- "index" => @index.to_s
107
- ]
117
+ "index" => @placeholder_expander.expand(@index)
118
+ ]
119
+ # TODO: parse different source types as expected: KVP, JSON, TEXT
120
+ if @all_items
121
+ splunk_object["event"] = record
122
+ else
123
+ splunk_object["event"] = record["message"]
124
+ end
108
125
  json_event = splunk_object.to_json
109
- #$log.debug "Generated JSON(#{json_event.class.to_s}): #{json_event.to_s}"
110
- #$log.debug "format: returning: #{[tag, record].to_json.to_s}"
126
+ #log.debug "Generated JSON(#{json_event.class.to_s}): #{json_event.to_s}"
127
+ #log.debug "format: returning: #{[tag, record].to_json.to_s}"
111
128
  json_event
112
129
  end
113
130
 
@@ -122,7 +139,7 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
122
139
  ##
123
140
  ## NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
124
141
  def write(chunk)
125
- $log.debug "splunk-http-eventcollector(write) called"
142
+ log.trace "splunk-http-eventcollector(write) called"
126
143
 
127
144
  # Break the concatenated string of JSON-formatted events into an Array
128
145
  split_chunk = chunk.read.split("}{").each do |x|
@@ -130,33 +147,33 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
130
147
  x.prepend("{") unless x.start_with?("{")
131
148
  x << "}" unless x.end_with?("}")
132
149
  end
133
- $log.debug "Pushing #{numfmt(split_chunk.size)} events (" +
150
+ log.debug "Pushing #{numfmt(split_chunk.size)} events (" +
134
151
  "#{numfmt(chunk.read.bytesize)} bytes) to Splunk."
135
152
  # If fluentd is pushing too much data to Splunk at once, split up the payload
136
153
  # Don't care about the number of events so much as the POST size (bytes)
137
154
  #if split_chunk.size > @batch_event_limit
138
- # $log.warn "Fluentd is attempting to push #{numfmt(split_chunk.size)} " +
155
+ # log.warn "Fluentd is attempting to push #{numfmt(split_chunk.size)} " +
139
156
  # "events in a single push to Splunk. The configured limit is " +
140
157
  # "#{numfmt(@batch_event_limit)}."
141
158
  #end
142
159
  if chunk.read.bytesize > @batch_size_limit
143
- $log.warn "Fluentd is attempting to push #{numfmt(chunk.read.bytesize)} " +
160
+ log.warn "Fluentd is attempting to push #{numfmt(chunk.read.bytesize)} " +
144
161
  "bytes in a single push to Splunk. The configured limit is " +
145
162
  "#{numfmt(@batch_size_limit)} bytes."
146
163
  newbuffer = Array.new
147
164
  split_chunk_counter = 0
148
165
  split_chunk.each do |c|
149
166
  split_chunk_counter = split_chunk_counter + 1
150
- #$log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
167
+ #log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
151
168
  # "newbuffer.bytesize=#{numfmt(newbuffer.join.bytesize)} + " +
152
169
  # "c.bytesize=#{numfmt(c.bytesize)} ????"
153
170
  if newbuffer.join.bytesize + c.bytesize < @batch_size_limit
154
- #$log.debug "Appended!"
171
+ #log.debug "Appended!"
155
172
  newbuffer << c
156
173
  else
157
174
  # Reached the limit - push the current newbuffer.join, and reset
158
- #$log.debug "Would exceed limit. Flushing newbuffer and continuing."
159
- $log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
175
+ #log.debug "Would exceed limit. Flushing newbuffer and continuing."
176
+ log.debug "(#{numfmt(split_chunk_counter)}/#{numfmt(split_chunk.size)}) " +
160
177
  "newbuffer.bytesize=#{numfmt(newbuffer.join.bytesize)} + " +
161
178
  "c.bytesize=#{numfmt(c.bytesize)} > #{numfmt(@batch_size_limit)}, " +
162
179
  "flushing current buffer to Splunk."
@@ -175,15 +192,15 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
175
192
  def push_buffer(body)
176
193
  post = Net::HTTP::Post.new @splunk_uri.request_uri
177
194
  post.body = body
178
- $log.debug "POST #{@splunk_uri}"
195
+ log.debug "POST #{@splunk_uri}"
179
196
  if @test_mode
180
- $log.debug "TEST_MODE Payload: #{body}"
197
+ log.debug "TEST_MODE Payload: #{body}"
181
198
  return
182
199
  end
183
200
  # retry up to :post_retry_max times
184
201
  1.upto(@post_retry_max) do |c|
185
202
  response = @http.request @splunk_uri, post
186
- $log.debug "=>(#{c}/#{numfmt(@post_retry_max)}) #{response.code} " +
203
+ log.debug "=>(#{c}/#{numfmt(@post_retry_max)}) #{response.code} " +
187
204
  "(#{response.message})"
188
205
  # TODO check the actual server response too (it's JSON)
189
206
  if response.code == "200" # and...
@@ -192,18 +209,18 @@ class SplunkHTTPEventcollectorOutput < BufferedOutput
192
209
  # TODO check 40X response within post_retry_max and retry
193
210
  elsif response.code.match(/^50/) and c < @post_retry_max
194
211
  # retry
195
- $log.warn "#{@splunk_uri}: Server error #{response.code} (" +
212
+ log.warn "#{@splunk_uri}: Server error #{response.code} (" +
196
213
  "#{response.message}). Retrying in #{@post_retry_interval} " +
197
214
  "seconds.\n#{response.body}"
198
215
  sleep @post_retry_interval
199
216
  next
200
217
  elsif response.code.match(/^40/)
201
218
  # user error
202
- $log.error "#{@splunk_uri}: #{response.code} (#{response.message})\n#{response.body}"
219
+ log.error "#{@splunk_uri}: #{response.code} (#{response.message})\n#{response.body}"
203
220
  break
204
221
  elsif c < @post_retry_max
205
222
  # retry
206
- $log.debug "#{@splunk_uri}: Retrying..."
223
+ log.debug "#{@splunk_uri}: Retrying..."
207
224
  sleep @post_retry_interval
208
225
  next
209
226
  else
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-splunk-http-eventcollector
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Bryce Chidester
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-10-04 00:00:00.000000000 Z
11
+ date: 2017-02-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: test-unit
@@ -52,6 +52,20 @@ dependencies:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: '2.9'
55
+ - !ruby/object:Gem::Dependency
56
+ name: fluent-mixin-rewrite-tag-name
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: 0.1.0
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: 0.1.0
55
69
  description: Splunk output plugin (HTTP Event Collector) for Fluentd event collector
56
70
  email:
57
71
  - bryce.chidester@calyptix.com
@@ -72,7 +86,7 @@ files:
72
86
  - test/plugin/test_out_splunk-http-eventcollector.rb
73
87
  homepage: https://github.com/brycied00d/fluent-plugin-splunk-http-eventcollector
74
88
  licenses:
75
- - BSD 2-clause
89
+ - BSD-2-Clause
76
90
  metadata: {}
77
91
  post_install_message:
78
92
  rdoc_options: []
@@ -90,7 +104,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
90
104
  version: '0'
91
105
  requirements: []
92
106
  rubyforge_project:
93
- rubygems_version: 2.4.5.1
107
+ rubygems_version: 2.6.10
94
108
  signing_key:
95
109
  specification_version: 4
96
110
  summary: Splunk output plugin for Fluentd