fluent-plugin-splunkhec 1.7 → 2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 64bafb9078216f8761e88b562441c73861fa0b20
4
- data.tar.gz: 79933b686c1619ac6ddd79789df0034fd2ec8403
3
+ metadata.gz: '058b51ece316507dd70cc84bd3ec0c32372e1f8a'
4
+ data.tar.gz: bff476df06f9a86433c9b10f74975a5b3d69c301
5
5
  SHA512:
6
- metadata.gz: 89d43832331ea520882e0129cddaea6942a0cc1c9453d3a4ea6997d174f47791f86663797c8966cdcf1785b370354f12945ead0933c02fbaedda34c551cefa0b
7
- data.tar.gz: 9ffe57c6011cdf2a349695a05534513c2d216eba7e984a6d980ab43a63cfd79825bf48ed11877aa3d265d18a3abe48435faba86d6ccbe222245885d21c197545
6
+ metadata.gz: ced5a57eeb88c6c9ea26bc57db6392f25abde00c89f85539b2a5fad3533972816ba35a4d90b22b3dde14b3b391b10a07ff0ae02782de5c783c5dccd092c74a79
7
+ data.tar.gz: 150458e3fa0c0b39cab8cc9f4c84bf231f45e70cdbe7344a9d610450066595d16ccdaecaa0dc74e133edcdf17a0d65a915458ce5adc2f4d3265154f187aa1599
@@ -1,3 +1,15 @@
1
+ ## 2.0
2
+
3
+ Migrate to use FluentD v1 API. It doesn't support backwards compatibility.
4
+
5
+ ## 1.9
6
+
7
+ Reverted source and sourcetpye settings. They now reflect the README.
8
+
9
+ ## 1.8
10
+
11
+ - Add expand function used in ES plugin an get variables from kubernetes tags in fluent.conf
12
+
1
13
  ## 1.7
2
14
 
3
15
  - Fixed HTTP request (removed verify none)
@@ -49,4 +61,4 @@ Replaced RestClient for net/http.
49
61
 
50
62
  ## 0.9.0
51
63
 
52
- First version
64
+ First version
data/LICENSE CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2017 Coen Meerbeek
1
+ Copyright (c) 2018 Coen Meerbeek
2
2
 
3
3
  MIT License
4
4
 
data/README.md CHANGED
@@ -86,4 +86,4 @@ Specify that all events in a FluentD chunk should be sent in batch to Splunk. De
86
86
 
87
87
  ## Copyright
88
88
 
89
- Copyright (c) 2017 Coen Meerbeek. See [LICENSE](LICENSE) for details.
89
+ Copyright (c) 2018 Coen Meerbeek. See [LICENSE](LICENSE) for details.
@@ -4,19 +4,19 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |gem|
6
6
  gem.name = "fluent-plugin-splunkhec"
7
- gem.version = "1.7"
7
+ gem.version = "2.0"
8
8
  gem.authors = "Coen Meerbeek"
9
9
  gem.email = "cmeerbeek@gmail.com"
10
10
  gem.description = %q{Output plugin for the Splunk HTTP Event Collector.}
11
11
  gem.homepage = "https://github.com/cmeerbeek/fluent-plugin-splunkhec"
12
12
  gem.summary = %q{This plugin allows you to sent events to the Splunk HTTP Event Collector.}
13
-
13
+
14
14
  gem.files = `git ls-files`.split($\)
15
15
  gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
16
16
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
17
17
  gem.require_paths = ["lib"]
18
-
19
- gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
18
+
19
+ gem.add_dependency "fluentd", [">= 0.14.15", "< 2"]
20
20
  gem.add_dependency "yajl-ruby", '>= 1.3.0'
21
21
  gem.add_development_dependency "rake", '~> 0.9', '>= 0.9.2'
22
22
  gem.add_development_dependency "test-unit", '~> 3.1', '>= 3.1.0'
@@ -1,11 +1,15 @@
1
- require 'fluent/output'
1
+ require 'fluent/plugin/output'
2
2
  require 'net/http'
3
3
  require 'yajl/json_gem'
4
4
 
5
- module Fluent
6
- class SplunkHECOutput < BufferedOutput
5
+ module Fluent::Plugin
6
+ class SplunkHECOutput < Output
7
7
  Fluent::Plugin.register_output('splunkhec', self)
8
8
 
9
+ helpers :compat_parameters, :event_emitter
10
+
11
+ DEFAULT_BUFFER_TYPE = "memory"
12
+
9
13
  # Primary Splunk HEC configuration parameters
10
14
  config_param :host, :string, :default => 'localhost'
11
15
  config_param :protocol, :string, :default => 'http'
@@ -15,19 +19,24 @@ module Fluent
15
19
  # Splunk event parameters
16
20
  config_param :index, :string, :default => 'main'
17
21
  config_param :event_host, :string, :default => nil
18
- config_param :source, :string, :default => 'tag'
19
- config_param :sourcetype, :string, :default => 'fluentd'
22
+ config_param :source, :string, :default => 'fluentd'
23
+ config_param :sourcetype, :string, :default => 'tag'
20
24
  config_param :send_event_as_json, :bool, :default => false
21
25
  config_param :usejson, :bool, :default => true
22
26
  config_param :send_batched_events, :bool, :default => false
23
27
 
28
+ config_section :buffer do
29
+ config_set_default :@type, DEFAULT_BUFFER_TYPE
30
+ end
31
+
24
32
  # This method is called before starting.
25
33
  # Here we construct the Splunk HEC URL to POST data to
26
34
  # If the configuration is invalid, raise Fluent::ConfigError.
27
35
  def configure(conf)
36
+ compat_parameters_convert(conf, :buffer)
28
37
  super
29
38
  @splunk_url = @protocol + '://' + @host + ':' + @port + '/services/collector/event'
30
- log.info 'splunkhec: sent data to ' + @splunk_url
39
+ log.info 'splunkhec: sending data to ' + @splunk_url
31
40
 
32
41
  if conf['event_host'] == nil
33
42
  begin
@@ -36,6 +45,7 @@ module Fluent
36
45
  @event_host = 'unknown'
37
46
  end
38
47
  end
48
+ @packer = Fluent::Engine.msgpack_factory.packer
39
49
  end
40
50
 
41
51
  def start
@@ -46,16 +56,67 @@ module Fluent
46
56
  super
47
57
  end
48
58
 
59
+ def formatted_to_msgpack_binary?
60
+ true
61
+ end
62
+
63
+ def multi_workers_ready?
64
+ true
65
+ end
66
+
49
67
  # This method is called when an event reaches to Fluentd.
50
68
  # Use msgpack to serialize the object.
51
69
  def format(tag, time, record)
52
- [tag, time, record].to_msgpack
70
+ @packer.pack([tag, time, record]).to_s
71
+ end
72
+
73
+ def expand_param(param, tag, time, record)
74
+ # check for '${ ... }'
75
+ # yes => `eval`
76
+ # no => return param
77
+ return param if (param =~ /\${.+}/).nil?
78
+
79
+ # check for 'tag_parts[]'
80
+ # separated by a delimiter (default '.')
81
+ tag_parts = tag.split(@delimiter) unless (param =~ /tag_parts\[.+\]/).nil? || tag.nil?
82
+
83
+ # pull out section between ${} then eval
84
+ inner = param.clone
85
+ while inner.match(/\${.+}/)
86
+ to_eval = inner.match(/\${(.+?)}/){$1}
87
+
88
+ if !(to_eval =~ /record\[.+\]/).nil? && record.nil?
89
+ return to_eval
90
+ elsif !(to_eval =~/tag_parts\[.+\]/).nil? && tag_parts.nil?
91
+ return to_eval
92
+ elsif !(to_eval =~/time/).nil? && time.nil?
93
+ return to_eval
94
+ else
95
+ inner.sub!(/\${.+?}/, eval( to_eval ))
96
+ end
97
+ end
98
+ inner
53
99
  end
54
100
 
55
101
  # Loop through all records and sent them to Splunk
56
102
  def write(chunk)
57
103
  body = ''
58
104
  chunk.msgpack_each {|(tag,time,record)|
105
+
106
+ # define index and sourcetype dynamically
107
+ begin
108
+ index = expand_param(@index, tag, time, record)
109
+ sourcetype = expand_param(@sourcetype, tag, time, record)
110
+ event_host = expand_param(@event_host, tag, time, record)
111
+ token = expand_param(@token, tag, time, record)
112
+ rescue => e
113
+ # handle dynamic parameters misconfigurations
114
+ router.emit_error_event(tag, time, record, e)
115
+ next
116
+ end
117
+ log.debug "routing event from #{event_host} to #{index} index"
118
+ log.debug "expanded token #{token}"
119
+
59
120
  # Parse record to Splunk event format
60
121
  case record
61
122
  when Integer
@@ -70,41 +131,42 @@ module Fluent
70
131
  event = record
71
132
  end
72
133
 
73
- source = @source == 'tag' ? tag : @source
134
+ sourcetype = @sourcetype == 'tag' ? tag : @sourcetype
74
135
 
75
136
  # Build body for the POST request
76
137
  if !@usejson
77
138
  event = record["time"]+ " " + record["message"].to_json.gsub(/^"|"$/,"")
78
- body << '{"time":"'+ DateTime.parse(record["time"]).strftime("%Q") +'", "event":"' + event + '", "sourcetype" :"' + sourcetype + '", "source" :"' + @source + '", "index" :"' + @index + '", "host" : "' + @event_host + '"}'
139
+ body << '{"time":"'+ DateTime.parse(record["time"]).strftime("%Q") +'", "event":"' + event + '", "sourcetype" :"' + sourcetype + '", "source" :"' + @source + '", "index" :"' + index + '", "host" : "' + event_host + '"}'
79
140
  elsif @send_event_as_json
80
- body << '{"time" :' + time.to_s + ', "event" :' + event + ', "sourcetype" :"' + sourcetype + '", "source" :"' + source + '", "index" :"' + @index + '", "host" : "' + @event_host + '"}'
141
+ body << '{"time" :' + time.to_s + ', "event" :' + event + ', "sourcetype" :"' + sourcetype + '", "source" :"' + source + '", "index" :"' + index + '", "host" : "' + event_host + '"}'
81
142
  else
82
- body << '{"time" :' + time.to_s + ', "event" :"' + event + '", "sourcetype" :"' + sourcetype + '", "source" :"' + source + '", "index" :"' + @index + '", "host" : "' + @event_host + '"}'
143
+ body << '{"time" :' + time.to_s + ', "event" :"' + event + '", "sourcetype" :"' + sourcetype + '", "source" :"' + source + '", "index" :"' + index + '", "host" : "' + event_host + '"}'
83
144
  end
84
145
 
85
146
  if @send_batched_events
86
147
  body << "\n"
87
148
  else
88
- send_to_splunk(body)
149
+ send_to_splunk(body, token)
89
150
  body = ''
90
151
  end
91
152
  }
92
153
 
93
154
  if @send_batched_events
94
- send_to_splunk(body)
155
+ send_to_splunk(body, token)
95
156
  end
96
157
  end
97
158
 
98
- def send_to_splunk(body)
159
+ def send_to_splunk(body, token)
99
160
  log.debug "splunkhec: " + body + "\n"
100
161
 
101
162
  uri = URI(@splunk_url)
102
163
 
103
164
  # Create client
104
165
  http = Net::HTTP.new(uri.host, uri.port)
166
+ http.set_debug_output(log.debug)
105
167
 
106
168
  # Create request
107
- req = Net::HTTP::Post.new(uri, "Content-Type" => "application/json; charset=utf-8", "Authorization" => "Splunk #{@token}")
169
+ req = Net::HTTP::Post.new(uri, "Content-Type" => "application/json; charset=utf-8", "Authorization" => "Splunk #{token}")
108
170
  req.body = body
109
171
 
110
172
  # Handle SSL
@@ -14,18 +14,9 @@ require "test/unit"
14
14
  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
15
15
  $LOAD_PATH.unshift(File.dirname(__FILE__))
16
16
  require "fluent/test"
17
- unless ENV.has_key?("VERBOSE")
18
- nulllogger = Object.new
19
- nulllogger.instance_eval {|obj|
20
- def method_missing(method, *args)
21
- #pass
22
- end
23
- }
24
- $log = nulllogger
25
- end
17
+ require "fluent/test/driver/output"
26
18
 
27
19
  require "fluent/plugin/out_splunkhec"
28
20
 
29
21
  class Test::Unit::TestCase
30
22
  end
31
-
@@ -1,3 +1,4 @@
1
+ # coding: utf-8
1
2
  require 'helper'
2
3
  require 'webmock/test_unit'
3
4
 
@@ -26,7 +27,7 @@ class SplunkHECOutputTest < Test::Unit::TestCase
26
27
  ]
27
28
 
28
29
  def create_driver_splunkhec(conf = CONFIG)
29
- Fluent::Test::BufferedOutputTestDriver.new(Fluent::SplunkHECOutput).configure(conf)
30
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::SplunkHECOutput).configure(conf)
30
31
  end
31
32
 
32
33
  def setup
@@ -48,8 +49,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
48
49
  assert_equal '8088', d.instance.port
49
50
  assert_equal 'main', d.instance.index
50
51
  assert_equal `hostname`.delete!("\n"), d.instance.event_host
51
- assert_equal 'fluentd', d.instance.sourcetype
52
- assert_equal 'tag', d.instance.source
52
+ assert_equal 'tag', d.instance.sourcetype
53
+ assert_equal 'fluentd', d.instance.source
53
54
  assert_equal false, d.instance.send_event_as_json
54
55
  assert_equal true, d.instance.usejson
55
56
  assert_equal false, d.instance.send_batched_events
@@ -85,8 +86,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
85
86
  })
86
87
 
87
88
  d = create_driver_splunkhec(CONFIG + %[sourcetype #{sourcetype}])
88
- d.run do
89
- d.emit(record, time)
89
+ d.run(default_tag: 'test') do
90
+ d.feed(time, record)
90
91
  end
91
92
 
92
93
  assert_requested(splunk_request)
@@ -96,8 +97,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
96
97
  splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'sourcetype' => 'test'}))
97
98
 
98
99
  d = create_driver_splunkhec(CONFIG + %[sourcetype test])
99
- d.run do
100
- d.emit({'message' => 'data'}, 123456)
100
+ d.run(default_tag: 'test') do
101
+ d.feed(123456, {'message' => 'data'})
101
102
  end
102
103
 
103
104
  assert_requested(splunk_request)
@@ -108,8 +109,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
108
109
  splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'event' => record.to_json}))
109
110
 
110
111
  d = create_driver_splunkhec(CONFIG + %[send_event_as_json false])
111
- d.run do
112
- d.emit(record)
112
+ d.run(default_tag: 'test') do
113
+ d.feed(record)
113
114
  end
114
115
 
115
116
  assert_requested(splunk_request)
@@ -124,8 +125,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
124
125
  .with(body: hash_including({'time' => log_time_millis, 'event' => "#{log_time} #{log_event}"}))
125
126
 
126
127
  d = create_driver_splunkhec(CONFIG + %[usejson false])
127
- d.run do
128
- d.emit({'time' => log_time, 'message' => log_event})
128
+ d.run(default_tag: 'test') do
129
+ d.feed({'time' => log_time, 'message' => log_event})
129
130
  end
130
131
 
131
132
  assert_requested(splunk_request)
@@ -137,8 +138,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
137
138
  splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'event' => record}))
138
139
 
139
140
  d = create_driver_splunkhec(CONFIG + %[send_event_as_json true])
140
- d.run do
141
- d.emit(record)
141
+ d.run(default_tag: 'test') do
142
+ d.feed(record)
142
143
  end
143
144
 
144
145
  assert_requested(splunk_request)
@@ -154,9 +155,9 @@ class SplunkHECOutputTest < Test::Unit::TestCase
154
155
  send_event_as_json true
155
156
  send_batched_events true])
156
157
 
157
- d.run do
158
- d.emit(record1)
159
- d.emit(record2)
158
+ d.run(default_tag: 'test') do
159
+ d.feed(record1)
160
+ d.feed(record2)
160
161
  end
161
162
 
162
163
  assert_requested(splunk_request)
@@ -165,10 +166,10 @@ class SplunkHECOutputTest < Test::Unit::TestCase
165
166
  def test_should_raise_exception_when_splunk_returns_error_to_make_fluentd_retry_later
166
167
  stub_request(:any, SPLUNK_URL).to_return(status: 403, body: {'text' => 'Token disabled', 'code' => 1}.to_json)
167
168
 
168
- assert_raise Fluent::SplunkHECOutputError do
169
+ assert_raise Fluent::Plugin::SplunkHECOutputError do
169
170
  d = create_driver_splunkhec
170
- d.run do
171
- d.emit({'message' => 'data'})
171
+ d.run(default_tag: 'test') do
172
+ d.feed({'message' => 'data'})
172
173
  end
173
174
  end
174
175
  end
@@ -179,8 +180,8 @@ class SplunkHECOutputTest < Test::Unit::TestCase
179
180
  splunk_request = stub_request(:post, SPLUNK_URL).with(body: hash_including({'event' => {'message' => '©2017'}}))
180
181
 
181
182
  d = create_driver_splunkhec(CONFIG + %[send_event_as_json true])
182
- d.run do
183
- d.emit(record)
183
+ d.run(default_tag: 'test') do
184
+ d.feed(record)
184
185
  end
185
186
 
186
187
  assert_requested(splunk_request)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-splunkhec
3
3
  version: !ruby/object:Gem::Version
4
- version: '1.7'
4
+ version: '2.0'
5
5
  platform: ruby
6
6
  authors:
7
7
  - Coen Meerbeek
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-05-15 00:00:00.000000000 Z
11
+ date: 2019-02-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -16,7 +16,7 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 0.10.58
19
+ version: 0.14.15
20
20
  - - "<"
21
21
  - !ruby/object:Gem::Version
22
22
  version: '2'
@@ -26,7 +26,7 @@ dependencies:
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: 0.10.58
29
+ version: 0.14.15
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
32
  version: '2'
@@ -107,7 +107,6 @@ files:
107
107
  - ".travis.yml"
108
108
  - CHANGELOG.md
109
109
  - Gemfile
110
- - Gemfile.lock
111
110
  - LICENSE
112
111
  - README.md
113
112
  - Rakefile
@@ -135,7 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
135
134
  version: '0'
136
135
  requirements: []
137
136
  rubyforge_project:
138
- rubygems_version: 2.6.8
137
+ rubygems_version: 2.5.2.3
139
138
  signing_key:
140
139
  specification_version: 4
141
140
  summary: This plugin allows you to sent events to the Splunk HTTP Event Collector.
@@ -1,62 +0,0 @@
1
- PATH
2
- remote: .
3
- specs:
4
- fluent-plugin-splunkhec (1.5)
5
- fluentd (>= 0.10.58, < 2)
6
- yajl-ruby (>= 1.3.0)
7
-
8
- GEM
9
- remote: https://rubygems.org/
10
- specs:
11
- addressable (2.5.2)
12
- public_suffix (>= 2.0.2, < 4.0)
13
- cool.io (1.5.3)
14
- crack (0.4.3)
15
- safe_yaml (~> 1.0.0)
16
- dig_rb (1.0.1)
17
- fluentd (1.2.0)
18
- cool.io (>= 1.4.5, < 2.0.0)
19
- dig_rb (~> 1.0.0)
20
- http_parser.rb (>= 0.5.1, < 0.7.0)
21
- msgpack (>= 0.7.0, < 2.0.0)
22
- serverengine (>= 2.0.4, < 3.0.0)
23
- sigdump (~> 0.2.2)
24
- strptime (>= 0.2.2, < 1.0.0)
25
- tzinfo (~> 1.0)
26
- tzinfo-data (~> 1.0)
27
- yajl-ruby (~> 1.0)
28
- hashdiff (0.3.7)
29
- http_parser.rb (0.6.0)
30
- msgpack (1.2.4)
31
- power_assert (1.1.1)
32
- public_suffix (3.0.2)
33
- rake (0.9.6)
34
- safe_yaml (1.0.4)
35
- serverengine (2.0.6)
36
- sigdump (~> 0.2.2)
37
- sigdump (0.2.4)
38
- strptime (0.2.3)
39
- test-unit (3.2.7)
40
- power_assert
41
- thread_safe (0.3.6)
42
- tzinfo (1.2.5)
43
- thread_safe (~> 0.1)
44
- tzinfo-data (1.2018.5)
45
- tzinfo (>= 1.0.0)
46
- webmock (3.4.1)
47
- addressable (>= 2.3.6)
48
- crack (>= 0.3.2)
49
- hashdiff
50
- yajl-ruby (1.4.0)
51
-
52
- PLATFORMS
53
- ruby
54
-
55
- DEPENDENCIES
56
- fluent-plugin-splunkhec!
57
- rake (~> 0.9, >= 0.9.2)
58
- test-unit (~> 3.1, >= 3.1.0)
59
- webmock (>= 3.0)
60
-
61
- BUNDLED WITH
62
- 1.16.1