fluent-plugin-kubernetes-objects 1.0.0 → 1.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +6 -2
- data/Gemfile.lock +58 -43
- data/README.md +1 -1
- data/Rakefile +6 -6
- data/VERSION +1 -0
- data/fluent-plugin-kubernetes-objects.gemspec +19 -29
- data/lib/fluent/plugin/in_kubernetes_objects.rb +109 -82
- data/test/fluent/plugin/in_kubernetes_objects_test.rb +34 -20
- data/test/test_helper.rb +46 -36
- metadata +39 -25
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 23311c5c8d08571e1421a02d990106536269c56c13a7010eb33ff468374524d7
|
4
|
+
data.tar.gz: 30a04f002df00e89d8d7ba000f060d6d410d99ce6d1b9301d46b8309ef330561
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6dea9c0003540c9ed06b715c64cb7ba4fdce1fa538d3e0b5e11a3c0ab23b4b254f80074bd9872d26600e3b6b2fe181ea221ba33fac91c91b02925b2515a48c34
|
7
|
+
data.tar.gz: db71c9f00e2ff3b2d35f2340b95b7a4f826fe38f2ed3bc771d444296ea80748c0127b5b38b8b1d677253a9af9e7c420180ba1bb43f589f99141b7c412e056d42
|
data/Gemfile
CHANGED
@@ -1,6 +1,10 @@
|
|
1
|
-
source
|
1
|
+
source 'https://rubygems.org'
|
2
2
|
|
3
|
-
|
3
|
+
group :test do
|
4
|
+
gem 'simplecov', require: false
|
5
|
+
end
|
6
|
+
|
7
|
+
git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
|
4
8
|
|
5
9
|
# Specify your gem's dependencies in fluent-plugin-kubernetes_objects_input.gemspec
|
6
10
|
gemspec
|
data/Gemfile.lock
CHANGED
@@ -1,91 +1,106 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-kubernetes-objects (1.
|
5
|
-
fluentd (~> 1.
|
6
|
-
|
4
|
+
fluent-plugin-kubernetes-objects (1.1.4)
|
5
|
+
fluentd (~> 1.9.1)
|
6
|
+
http_parser.rb (= 0.5.3)
|
7
|
+
kubeclient (~> 4.6.0)
|
7
8
|
|
8
9
|
GEM
|
9
10
|
remote: https://rubygems.org/
|
10
11
|
specs:
|
11
|
-
addressable (2.
|
12
|
+
addressable (2.6.0)
|
12
13
|
public_suffix (>= 2.0.2, < 4.0)
|
13
|
-
|
14
|
+
concurrent-ruby (1.1.6)
|
15
|
+
cool.io (1.6.0)
|
14
16
|
crack (0.4.3)
|
15
17
|
safe_yaml (~> 1.0.0)
|
16
|
-
|
17
|
-
domain_name (0.5.
|
18
|
+
docile (1.3.1)
|
19
|
+
domain_name (0.5.20190701)
|
18
20
|
unf (>= 0.0.5, < 1.0.0)
|
19
|
-
|
21
|
+
ffi (1.12.2)
|
22
|
+
ffi-compiler (1.0.1)
|
23
|
+
ffi (>= 1.0.0)
|
24
|
+
rake
|
25
|
+
fluentd (1.9.2)
|
20
26
|
cool.io (>= 1.4.5, < 2.0.0)
|
21
|
-
dig_rb (~> 1.0.0)
|
22
27
|
http_parser.rb (>= 0.5.1, < 0.7.0)
|
23
|
-
msgpack (>=
|
28
|
+
msgpack (>= 1.3.1, < 2.0.0)
|
24
29
|
serverengine (>= 2.0.4, < 3.0.0)
|
25
30
|
sigdump (~> 0.2.2)
|
26
31
|
strptime (>= 0.2.2, < 1.0.0)
|
27
|
-
tzinfo (
|
32
|
+
tzinfo (>= 1.0, < 3.0)
|
28
33
|
tzinfo-data (~> 1.0)
|
29
34
|
yajl-ruby (~> 1.0)
|
30
|
-
hashdiff (0.3.
|
31
|
-
http (
|
35
|
+
hashdiff (0.3.8)
|
36
|
+
http (4.3.0)
|
32
37
|
addressable (~> 2.3)
|
33
38
|
http-cookie (~> 1.0)
|
34
|
-
http-form_data (~>
|
35
|
-
|
39
|
+
http-form_data (~> 2.2)
|
40
|
+
http-parser (~> 1.2.0)
|
41
|
+
http-accept (1.7.0)
|
36
42
|
http-cookie (1.0.3)
|
37
43
|
domain_name (~> 0.5)
|
38
|
-
http-form_data (
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
44
|
+
http-form_data (2.2.0)
|
45
|
+
http-parser (1.2.1)
|
46
|
+
ffi-compiler (>= 1.0, < 2.0)
|
47
|
+
http_parser.rb (0.5.3)
|
48
|
+
json (2.3.1)
|
49
|
+
kubeclient (4.6.0)
|
50
|
+
http (>= 3.0, < 5.0)
|
51
|
+
recursive-open-struct (~> 1.0, >= 1.0.4)
|
43
52
|
rest-client (~> 2.0)
|
44
|
-
mime-types (3.1)
|
53
|
+
mime-types (3.3.1)
|
45
54
|
mime-types-data (~> 3.2015)
|
46
|
-
mime-types-data (3.
|
55
|
+
mime-types-data (3.2019.1009)
|
47
56
|
minitest (5.11.3)
|
48
|
-
msgpack (1.
|
57
|
+
msgpack (1.3.3)
|
49
58
|
netrc (0.11.0)
|
50
|
-
power_assert (1.1.
|
51
|
-
public_suffix (3.0.
|
52
|
-
rake (
|
53
|
-
recursive-open-struct (1.0
|
54
|
-
rest-client (2.0
|
59
|
+
power_assert (1.1.3)
|
60
|
+
public_suffix (3.0.3)
|
61
|
+
rake (13.0.1)
|
62
|
+
recursive-open-struct (1.1.0)
|
63
|
+
rest-client (2.1.0)
|
64
|
+
http-accept (>= 1.7.0, < 2.0)
|
55
65
|
http-cookie (>= 1.0.2, < 2.0)
|
56
66
|
mime-types (>= 1.16, < 4.0)
|
57
67
|
netrc (~> 0.8)
|
58
68
|
safe_yaml (1.0.4)
|
59
|
-
serverengine (2.
|
69
|
+
serverengine (2.2.1)
|
60
70
|
sigdump (~> 0.2.2)
|
61
71
|
sigdump (0.2.4)
|
72
|
+
simplecov (0.16.1)
|
73
|
+
docile (~> 1.1)
|
74
|
+
json (>= 1.8, < 3)
|
75
|
+
simplecov-html (~> 0.10.0)
|
76
|
+
simplecov-html (0.10.2)
|
62
77
|
strptime (0.2.3)
|
63
|
-
test-unit (3.
|
78
|
+
test-unit (3.3.0)
|
64
79
|
power_assert
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
tzinfo-data (1.2018.3)
|
80
|
+
tzinfo (2.0.1)
|
81
|
+
concurrent-ruby (~> 1.0)
|
82
|
+
tzinfo-data (1.2019.3)
|
69
83
|
tzinfo (>= 1.0.0)
|
70
84
|
unf (0.1.4)
|
71
85
|
unf_ext
|
72
|
-
unf_ext (0.0.7.
|
73
|
-
webmock (3.
|
86
|
+
unf_ext (0.0.7.6)
|
87
|
+
webmock (3.5.1)
|
74
88
|
addressable (>= 2.3.6)
|
75
89
|
crack (>= 0.3.2)
|
76
90
|
hashdiff
|
77
|
-
yajl-ruby (1.
|
91
|
+
yajl-ruby (1.4.1)
|
78
92
|
|
79
93
|
PLATFORMS
|
80
94
|
ruby
|
81
95
|
|
82
96
|
DEPENDENCIES
|
83
|
-
bundler (~>
|
97
|
+
bundler (~> 2.0)
|
84
98
|
fluent-plugin-kubernetes-objects!
|
85
|
-
minitest (~> 5.
|
86
|
-
rake (
|
87
|
-
|
88
|
-
|
99
|
+
minitest (~> 5.11)
|
100
|
+
rake (>= 12.0)
|
101
|
+
simplecov
|
102
|
+
test-unit (~> 3.3)
|
103
|
+
webmock (~> 3.5)
|
89
104
|
|
90
105
|
BUNDLED WITH
|
91
|
-
1.
|
106
|
+
2.1.4
|
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
[
|
1
|
+
[![CircleCI](https://circleci.com/gh/git-lfs/git-lfs.svg?style=shield&circle-token=856152c2b02bfd236f54d21e1f581f3e4ebf47ad)](https://circleci.com/gh/splunk/fluent-plugin-kubernetes-objects)
|
2
2
|
# fluent-plugin-kubernetes-objects
|
3
3
|
|
4
4
|
[Fluentd](https://fluentd.org/) input plugin to collect [objects](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/) in a kubernetes cluster.
|
data/Rakefile
CHANGED
@@ -1,10 +1,10 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require 'bundler/gem_tasks'
|
2
|
+
require 'rake/testtask'
|
3
3
|
|
4
4
|
Rake::TestTask.new(:test) do |t|
|
5
|
-
t.libs <<
|
6
|
-
t.libs <<
|
7
|
-
t.test_files = FileList[
|
5
|
+
t.libs << 'test'
|
6
|
+
t.libs << 'lib'
|
7
|
+
t.test_files = FileList['test/**/*_test.rb']
|
8
8
|
end
|
9
9
|
|
10
|
-
task :
|
10
|
+
task default: :test
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
1.1.4
|
@@ -1,43 +1,33 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
|
-
spec.name =
|
3
|
-
spec.version =
|
4
|
-
spec.authors = [
|
5
|
-
spec.email = [
|
2
|
+
spec.name = 'fluent-plugin-kubernetes-objects'
|
3
|
+
spec.version = File.read('VERSION')
|
4
|
+
spec.authors = ['Splunk Inc.']
|
5
|
+
spec.email = ['DataEdge@splunk.com']
|
6
6
|
|
7
|
-
spec.summary =
|
8
|
-
spec.description =
|
9
|
-
spec.homepage =
|
10
|
-
spec.license =
|
7
|
+
spec.summary = 'Fluentd Plugin for Kubernetes Objects.'
|
8
|
+
spec.description = 'A Fluentd input plugin for collecting Kubernetes objects, e.g. pods, namespaces, events, etc. by pulling or watching.'
|
9
|
+
spec.homepage = 'https://github.com/splunk/fluent-plugin-kubernetes-objects'
|
10
|
+
spec.license = 'Apache-2.0'
|
11
11
|
|
12
|
-
|
13
|
-
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
14
|
-
# if spec.respond_to?(:metadata)
|
15
|
-
# spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
|
16
|
-
# else
|
17
|
-
# raise "RubyGems 2.0 or newer is required to protect against " \
|
18
|
-
# "public gem pushes."
|
19
|
-
# end
|
20
|
-
|
21
|
-
spec.require_paths = ["lib"]
|
12
|
+
spec.require_paths = ['lib']
|
22
13
|
spec.test_files = Dir.glob('test/**/**.rb')
|
23
14
|
spec.files = %w[
|
24
15
|
CODE_OF_CONDUCT.md README.md LICENSE
|
25
16
|
fluent-plugin-kubernetes-objects.gemspec
|
26
17
|
Gemfile Gemfile.lock
|
27
|
-
Rakefile
|
18
|
+
Rakefile VERSION
|
28
19
|
] + Dir.glob('lib/**/**').reject(&File.method(:directory?))
|
29
20
|
|
30
|
-
spec.bindir =
|
21
|
+
spec.bindir = 'exe'
|
31
22
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
32
|
-
|
33
23
|
spec.required_ruby_version = '>= 2.3.0'
|
24
|
+
spec.add_runtime_dependency 'fluentd', '~> 1.9.1'
|
25
|
+
spec.add_runtime_dependency 'kubeclient', '~> 4.6.0'
|
26
|
+
spec.add_runtime_dependency 'http_parser.rb', '= 0.5.3'
|
34
27
|
|
35
|
-
spec.
|
36
|
-
spec.
|
37
|
-
|
38
|
-
spec.add_development_dependency
|
39
|
-
spec.add_development_dependency
|
40
|
-
spec.add_development_dependency "test-unit", "~> 3.0" # required by fluent/test.rb
|
41
|
-
spec.add_development_dependency "minitest", "~> 5.0"
|
42
|
-
spec.add_development_dependency "webmock", "~> 3.2"
|
28
|
+
spec.add_development_dependency 'bundler', '~> 2.0'
|
29
|
+
spec.add_development_dependency 'minitest', '~> 5.11'
|
30
|
+
spec.add_development_dependency 'rake', '>= 12.0'
|
31
|
+
spec.add_development_dependency 'test-unit', '~> 3.3' # required by fluent/test.rb
|
32
|
+
spec.add_development_dependency 'webmock', '~> 3.5'
|
43
33
|
end
|
@@ -1,11 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require 'fluent/plugin/input'
|
4
4
|
require 'kubeclient'
|
5
5
|
|
6
6
|
module Fluent::Plugin
|
7
7
|
class KubernetesObjectsInput < Fluent::Plugin::Input
|
8
|
-
VERSION =
|
8
|
+
VERSION = '1.1.2'.freeze
|
9
9
|
|
10
10
|
Fluent::Plugin.register_input('kubernetes_objects', self)
|
11
11
|
|
@@ -72,21 +72,24 @@ module Fluent::Plugin
|
|
72
72
|
|
73
73
|
desc 'A selector to restrict the list of returned objects by fields.'
|
74
74
|
config_param :field_selector, :string, default: nil
|
75
|
+
|
76
|
+
desc 'The interval at which the objects will be watched.'
|
77
|
+
config_param :interval, :time, default: 15 * 60
|
75
78
|
end
|
76
79
|
|
77
80
|
config_section :storage do
|
78
81
|
# use memory by default
|
79
|
-
config_set_default :usage,
|
80
|
-
config_set_default :@type,
|
82
|
+
config_set_default :usage, 'checkpoints'
|
83
|
+
config_set_default :@type, 'local'
|
81
84
|
config_set_default :persistent, false
|
82
85
|
end
|
83
86
|
|
84
87
|
def configure(conf)
|
85
88
|
super
|
86
89
|
|
87
|
-
raise Fluent::ConfigError,
|
90
|
+
raise Fluent::ConfigError, 'At least one <pull> or <watch> is required, but found none.' if @pull_objects.empty? && @watch_objects.empty?
|
88
91
|
|
89
|
-
@storage = storage_create usage:
|
92
|
+
@storage = storage_create usage: 'checkpoints'
|
90
93
|
|
91
94
|
parse_tag
|
92
95
|
initialize_client
|
@@ -99,7 +102,6 @@ module Fluent::Plugin
|
|
99
102
|
end
|
100
103
|
|
101
104
|
def close
|
102
|
-
@watchers.each &:finish if @watchers
|
103
105
|
super
|
104
106
|
end
|
105
107
|
|
@@ -115,54 +117,69 @@ module Fluent::Plugin
|
|
115
117
|
[@tag_prefix, item_name, @tag_suffix].join
|
116
118
|
end
|
117
119
|
|
120
|
+
def init_with_kubeconfig()
|
121
|
+
options = {}
|
122
|
+
config = Kubeclient::Config.read @kubeconfig
|
123
|
+
current_context = config.context
|
124
|
+
|
125
|
+
@client = Kubeclient::Client.new(
|
126
|
+
current_context.api_endpoint,
|
127
|
+
current_context.api_version,
|
128
|
+
options.merge(
|
129
|
+
ssl_options: current_context.ssl_options,
|
130
|
+
auth_options: current_context.auth_options
|
131
|
+
)
|
132
|
+
)
|
133
|
+
end
|
134
|
+
|
118
135
|
def initialize_client
|
119
136
|
# mostly borrowed from Fluentd Kubernetes Metadata Filter Plugin
|
120
137
|
if @kubernetes_url.nil?
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
138
|
+
# Use Kubernetes default service account if we're in a pod.
|
139
|
+
env_host = ENV['KUBERNETES_SERVICE_HOST']
|
140
|
+
env_port = ENV['KUBERNETES_SERVICE_PORT']
|
141
|
+
if env_host && env_port
|
142
|
+
@kubernetes_url = "https://#{env_host}:#{env_port}/#{@api_version == 'v1' ? 'api' : 'apis'}"
|
143
|
+
end
|
127
144
|
end
|
128
145
|
|
129
|
-
raise Fluent::ConfigError,
|
146
|
+
raise Fluent::ConfigError, 'kubernetes url is not set' unless @kubernetes_url
|
130
147
|
|
131
148
|
# Use SSL certificate and bearer token from Kubernetes service account.
|
132
149
|
if Dir.exist?(@secret_dir)
|
133
|
-
|
134
|
-
|
150
|
+
secret_ca_file = File.join(@secret_dir, 'ca.crt')
|
151
|
+
secret_token_file = File.join(@secret_dir, 'token')
|
135
152
|
|
136
|
-
|
137
|
-
|
138
|
-
|
153
|
+
if @ca_file.nil? && File.exist?(secret_ca_file)
|
154
|
+
@ca_file = secret_ca_file
|
155
|
+
end
|
139
156
|
|
140
|
-
|
141
|
-
|
142
|
-
|
157
|
+
if @bearer_token_file.nil? && File.exist?(secret_token_file)
|
158
|
+
@bearer_token_file = secret_token_file
|
159
|
+
end
|
143
160
|
end
|
144
161
|
|
145
162
|
ssl_options = {
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
163
|
+
client_cert: @client_cert && OpenSSL::X509::Certificate.new(File.read(@client_cert)),
|
164
|
+
client_key: @client_key && OpenSSL::PKey::RSA.new(File.read(@client_key)),
|
165
|
+
ca_file: @ca_file,
|
166
|
+
verify_ssl: @insecure_ssl ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
|
150
167
|
}
|
151
168
|
|
152
169
|
auth_options = {}
|
153
170
|
auth_options[:bearer_token] = File.read(@bearer_token_file) if @bearer_token_file
|
154
171
|
|
155
172
|
@client = Kubeclient::Client.new(
|
156
|
-
|
157
|
-
|
158
|
-
|
173
|
+
@kubernetes_url, @api_version,
|
174
|
+
ssl_options: ssl_options,
|
175
|
+
auth_options: auth_options
|
159
176
|
)
|
160
177
|
|
161
178
|
begin
|
162
|
-
|
179
|
+
@client.api_valid?
|
163
180
|
rescue KubeException => kube_error
|
164
|
-
|
165
|
-
|
181
|
+
raise Fluent::ConfigError, "Invalid Kubernetes API #{@api_version} endpoint #{@kubernetes_url}: #{kube_error.message}"
|
182
|
+
end
|
166
183
|
end
|
167
184
|
|
168
185
|
def start_pullers
|
@@ -170,16 +187,7 @@ module Fluent::Plugin
|
|
170
187
|
end
|
171
188
|
|
172
189
|
def start_watchers
|
173
|
-
@
|
174
|
-
o = o.to_h.dup
|
175
|
-
o[:as] = :raw
|
176
|
-
resource_name = o.delete(:resource_name)
|
177
|
-
version = @storage.get(resource_name)
|
178
|
-
o[:resource_version] = version if version
|
179
|
-
@client.public_send("watch_#{resource_name}", o).tap { |watcher|
|
180
|
-
create_watcher_thread resource_name, watcher
|
181
|
-
}
|
182
|
-
end
|
190
|
+
@watch_objects.each(&method(:create_watcher_thread))
|
183
191
|
end
|
184
192
|
|
185
193
|
def create_pull_thread(conf)
|
@@ -189,49 +197,68 @@ module Fluent::Plugin
|
|
189
197
|
pull_interval = options.delete :interval
|
190
198
|
|
191
199
|
thread_create :"pull_#{resource_name}" do
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
200
|
+
tag = generate_tag resource_name
|
201
|
+
while thread_current_running?
|
202
|
+
log.debug "Going to pull #{resource_name}"
|
203
|
+
response = @client.public_send "get_#{resource_name}", options
|
204
|
+
now = Fluent::Engine.now
|
205
|
+
es = Fluent::MultiEventStream.new
|
206
|
+
|
207
|
+
# code copied from kubeclient
|
208
|
+
# kubeclient will create one open struct object for each item in the response,
|
209
|
+
# but this is totally unecessary in this plugin, thus we use as: :raw.
|
210
|
+
result = JSON.parse(response)
|
211
|
+
|
212
|
+
resource_version = result.fetch('resourceVersion') do
|
213
|
+
result.fetch('metadata', {})['resourceVersion']
|
214
|
+
end
|
215
|
+
|
216
|
+
update_op = if resource_version
|
217
|
+
->(item) { item['metadata'].update requestResourceVersion: resource_version }
|
218
|
+
else
|
219
|
+
->(item) {}
|
220
|
+
end
|
221
|
+
|
222
|
+
# result['items'] might be nil due to https://github.com/kubernetes/kubernetes/issues/13096
|
223
|
+
items = result['items'].to_a
|
224
|
+
log.debug { "Received #{items.size} #{resource_name}" }
|
225
|
+
items.each { |item| es.add now, item.tap(&update_op) }
|
226
|
+
router.emit_stream(tag, es)
|
227
|
+
|
228
|
+
sleep(pull_interval)
|
229
|
+
end
|
222
230
|
end
|
223
231
|
end
|
224
232
|
|
225
|
-
def create_watcher_thread(
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
233
|
+
def create_watcher_thread(conf)
|
234
|
+
options = conf.to_h.dup
|
235
|
+
options[:as] = :raw
|
236
|
+
resource_name = options[:resource_name]
|
237
|
+
version = @storage.get(resource_name)
|
238
|
+
if version
|
239
|
+
options[:resource_version] = version
|
240
|
+
else
|
241
|
+
options[:resource_version] = 0
|
242
|
+
end
|
243
|
+
|
244
|
+
thread_create :"watch_#{resource_name}" do
|
245
|
+
while thread_current_running?
|
246
|
+
@client.public_send("watch_#{resource_name}", options).tap do |watcher|
|
247
|
+
tag = generate_tag "#{resource_name}"
|
248
|
+
watcher.each do |entity|
|
249
|
+
begin
|
250
|
+
entity = JSON.parse(entity)
|
251
|
+
router.emit tag, Fluent::Engine.now, entity
|
252
|
+
options[:resource_version] = entity['object']['metadata']['resourceVersion']
|
253
|
+
@storage.put resource_name, entity['object']['metadata']['resourceVersion']
|
254
|
+
rescue => e
|
255
|
+
log.info "Got exception #{e} parsing entity #{entity}. Resetting watcher."
|
256
|
+
end
|
257
|
+
end
|
258
|
+
end
|
259
|
+
end
|
260
|
+
end
|
235
261
|
end
|
236
262
|
end
|
237
263
|
end
|
264
|
+
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require "test_helper"
|
2
|
+
require 'fluent/plugin/in_kubernetes_objects.rb'
|
2
3
|
|
3
4
|
describe Fluent::Plugin::KubernetesObjectsInput do
|
4
5
|
include Fluent::Test::Helpers
|
@@ -10,8 +11,10 @@ describe Fluent::Plugin::KubernetesObjectsInput do
|
|
10
11
|
}
|
11
12
|
|
12
13
|
it { expect(::Fluent::Plugin::KubernetesObjectsInput::VERSION).wont_be_nil }
|
14
|
+
puts 'Test: Version will not be nil'
|
13
15
|
|
14
16
|
it "should require at least one <pull> or <watch> section" do
|
17
|
+
puts 'Test: should require at least one <pull> or <watch> section'
|
15
18
|
expect{create_input_driver("kubernetes_url #{k8s_url}")}.must_raise Fluent::ConfigError
|
16
19
|
expect(create_input_driver(<<~CONF)).wont_be_nil
|
17
20
|
kubernetes_url #{k8s_url}
|
@@ -29,6 +32,7 @@ describe Fluent::Plugin::KubernetesObjectsInput do
|
|
29
32
|
|
30
33
|
describe "config: kubernetes_url" do
|
31
34
|
it "should read from environment variables by default" do
|
35
|
+
puts 'Test: should read from environment variables by default'
|
32
36
|
ENV['KUBERNETES_SERVICE_HOST'] = k8s_host
|
33
37
|
ENV['KUBERNETES_SERVICE_PORT'] = k8s_port
|
34
38
|
expect(create_input_driver(<<~CONF).instance.kubernetes_url).must_equal k8s_url
|
@@ -39,6 +43,7 @@ describe Fluent::Plugin::KubernetesObjectsInput do
|
|
39
43
|
end
|
40
44
|
|
41
45
|
it "should panic if not set" do
|
46
|
+
puts 'Test: should panic if not set'
|
42
47
|
ENV['KUBERNETES_SERVICE_HOST'] = nil
|
43
48
|
ENV['KUBERNETES_SERVICE_PORT'] = nil
|
44
49
|
expect{ create_input_driver(<<~CONF) }.must_raise Fluent::ConfigError
|
@@ -49,6 +54,7 @@ describe Fluent::Plugin::KubernetesObjectsInput do
|
|
49
54
|
end
|
50
55
|
|
51
56
|
it "should use pick the right path" do
|
57
|
+
puts 'Test: should use pick the right path'
|
52
58
|
ENV['KUBERNETES_SERVICE_HOST'] = k8s_host
|
53
59
|
ENV['KUBERNETES_SERVICE_PORT'] = k8s_port
|
54
60
|
expect(create_input_driver(<<~CONF).instance.kubernetes_url).must_equal k8s_url('apis')
|
@@ -62,6 +68,7 @@ describe Fluent::Plugin::KubernetesObjectsInput do
|
|
62
68
|
|
63
69
|
describe "emit events" do
|
64
70
|
it "can pull one resource" do
|
71
|
+
puts 'Test: can pull one resource'
|
65
72
|
d = create_input_driver(<<~CONF)
|
66
73
|
kubernetes_url #{k8s_url}
|
67
74
|
<pull>
|
@@ -76,6 +83,7 @@ describe Fluent::Plugin::KubernetesObjectsInput do
|
|
76
83
|
end
|
77
84
|
|
78
85
|
it "can pull multiple resources" do
|
86
|
+
puts 'Test: can pull multiple resources'
|
79
87
|
d = create_input_driver(<<~CONF)
|
80
88
|
kubernetes_url #{k8s_url}
|
81
89
|
<pull>
|
@@ -94,6 +102,7 @@ describe Fluent::Plugin::KubernetesObjectsInput do
|
|
94
102
|
end
|
95
103
|
|
96
104
|
it "can watch resources" do
|
105
|
+
puts 'Test: can watch resources'
|
97
106
|
d = create_input_driver(<<~CONF)
|
98
107
|
kubernetes_url #{k8s_url}
|
99
108
|
<watch>
|
@@ -101,33 +110,38 @@ describe Fluent::Plugin::KubernetesObjectsInput do
|
|
101
110
|
</watch>
|
102
111
|
CONF
|
103
112
|
|
113
|
+
stub_k8s_events params: {resourceVersion: "0"}
|
114
|
+
stub_k8s_events params: {resourceVersion: "6621683"}
|
115
|
+
|
104
116
|
d.run expect_emits: 1, timeout: 3
|
105
117
|
events = d.events
|
106
|
-
expect(events.all? { |e| e[0] == 'kubernetes.events
|
118
|
+
expect(events.all? { |e| e[0] == 'kubernetes.events'}).must_equal true
|
107
119
|
end
|
108
120
|
|
109
121
|
it "should use checkpoints for watching" do
|
122
|
+
puts 'Test: should use checkpoints for watching'
|
110
123
|
begin
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
124
|
+
require 'tempfile'
|
125
|
+
f = Tempfile.new("fluentd-k8s-objects-test", encoding: 'utf-8')
|
126
|
+
f.write('{"events": "123456"}')
|
127
|
+
f.close
|
128
|
+
|
129
|
+
d = create_input_driver(<<~CONF)
|
130
|
+
kubernetes_url #{k8s_url}
|
131
|
+
<storage>
|
132
|
+
path #{f.path}
|
133
|
+
</storage>
|
134
|
+
<watch>
|
135
|
+
resource_name events
|
136
|
+
</watch>
|
137
|
+
CONF
|
138
|
+
|
139
|
+
stub_k8s_events params: {resourceVersion: "123456"}
|
140
|
+
stub_k8s_events params: {resourceVersion: "6621683"}
|
141
|
+
|
142
|
+
d.run expect_emits: 1, timeout: 3
|
129
143
|
ensure
|
130
|
-
|
144
|
+
f.unlink
|
131
145
|
end
|
132
146
|
end
|
133
147
|
end
|
data/test/test_helper.rb
CHANGED
@@ -1,18 +1,19 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
require 'simplecov'
|
2
|
+
SimpleCov.start
|
3
3
|
|
4
|
-
|
4
|
+
$LOAD_PATH.unshift(File.expand_path('..', __dir__))
|
5
|
+
|
6
|
+
# suppress warning, when require the 'http' library shows circle require warning
|
5
7
|
# which is pretty annoying (kubeclient depends on http for watch_stream)
|
6
8
|
_verbose = $VERBOSE
|
7
9
|
$VERBOSE = nil
|
8
|
-
require "fluent/plugin/in_kubernetes_objects"
|
9
10
|
$VERBOSE = _verbose
|
10
11
|
|
11
|
-
require
|
12
|
-
require
|
13
|
-
require
|
14
|
-
require
|
15
|
-
require
|
12
|
+
require 'fluent/test'
|
13
|
+
require 'fluent/test/driver/input'
|
14
|
+
require 'fluent/test/helpers'
|
15
|
+
require 'minitest/autorun'
|
16
|
+
require 'webmock/minitest'
|
16
17
|
|
17
18
|
# make assertions from webmock available in minitest/spec
|
18
19
|
module Minitest::Expectations
|
@@ -21,20 +22,27 @@ module Minitest::Expectations
|
|
21
22
|
end
|
22
23
|
|
23
24
|
module PluginTestHelper
|
24
|
-
def k8s_host
|
25
|
-
|
26
|
-
|
25
|
+
def k8s_host
|
26
|
+
'127.0.0.1'
|
27
|
+
end
|
28
|
+
|
29
|
+
def k8s_port
|
30
|
+
'8001'
|
31
|
+
end
|
32
|
+
|
33
|
+
def k8s_url(path = 'api')
|
34
|
+
"https://#{k8s_host}:#{k8s_port}/#{path}"
|
35
|
+
end
|
27
36
|
|
28
37
|
def fluentd_conf_for(*lines)
|
29
|
-
basic_config = [
|
30
|
-
]
|
38
|
+
basic_config = []
|
31
39
|
(basic_config + lines).join("\n")
|
32
40
|
end
|
33
41
|
|
34
42
|
def create_input_driver(*configs)
|
35
|
-
Fluent::Test::Driver::Input.new(Fluent::Plugin::KubernetesObjectsInput).tap
|
43
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::KubernetesObjectsInput).tap do |d|
|
36
44
|
d.configure(fluentd_conf_for(*configs))
|
37
|
-
|
45
|
+
end
|
38
46
|
end
|
39
47
|
|
40
48
|
def stub_k8s_requests
|
@@ -49,47 +57,49 @@ module PluginTestHelper
|
|
49
57
|
end
|
50
58
|
|
51
59
|
def stub_k8s_api
|
52
|
-
open(File.expand_path('
|
60
|
+
File.open(File.expand_path('api.json', __dir__)).tap do |f|
|
53
61
|
stub_request(:get, k8s_url).to_return(body: f.read)
|
54
|
-
|
62
|
+
end.close
|
55
63
|
end
|
56
64
|
|
57
65
|
def stub_k8s_apis
|
58
|
-
open(File.expand_path('
|
66
|
+
File.open(File.expand_path('apis.json', __dir__)).tap do |f|
|
59
67
|
stub_request(:get, k8s_url('apis')).to_return(body: f.read)
|
60
|
-
|
68
|
+
end.close
|
61
69
|
end
|
62
70
|
|
63
71
|
def stub_k8s_v1
|
64
|
-
open(File.expand_path('
|
72
|
+
File.open(File.expand_path('v1.json', __dir__)).tap do |f|
|
65
73
|
stub_request(:get, "#{k8s_url}/v1").to_return(body: f.read)
|
66
|
-
|
74
|
+
end.close
|
67
75
|
end
|
68
76
|
|
69
77
|
def stub_k8s_namespaces
|
70
|
-
open(File.expand_path('
|
71
|
-
stub_request(:get, "#{k8s_url}/v1/namespaces").to_return(body: f.read
|
72
|
-
|
78
|
+
File.open(File.expand_path('namespaces.json', __dir__)).tap do |f|
|
79
|
+
stub_request(:get, "#{k8s_url}/v1/namespaces").to_return(body: f.read)
|
80
|
+
end.close
|
73
81
|
end
|
74
82
|
|
75
83
|
def stub_k8s_nodes
|
76
|
-
open(File.expand_path('
|
77
|
-
stub_request(:get, "#{k8s_url}/v1/nodes").to_return(body: f.read
|
78
|
-
|
84
|
+
File.open(File.expand_path('nodes.json', __dir__)).tap do |f|
|
85
|
+
stub_request(:get, "#{k8s_url}/v1/nodes").to_return(body: f.read)
|
86
|
+
end.close
|
79
87
|
end
|
80
88
|
|
81
89
|
def stub_k8s_pods
|
82
|
-
open(File.expand_path('
|
83
|
-
stub_request(:get, "#{k8s_url}/v1/pods").to_return(body: f.read
|
84
|
-
|
90
|
+
File.open(File.expand_path('pods.json', __dir__)).tap do |f|
|
91
|
+
stub_request(:get, "#{k8s_url}/v1/pods").to_return(body: f.read)
|
92
|
+
end.close
|
85
93
|
end
|
86
94
|
|
87
95
|
def stub_k8s_events(params: nil)
|
88
|
-
open(File.expand_path('
|
96
|
+
File.open(File.expand_path('events.json', __dir__)).tap do |f|
|
89
97
|
url = "#{k8s_url}/v1/watch/events"
|
90
|
-
url << '?' << params.map { |k, v|
|
91
|
-
stub_request(:get, url)
|
92
|
-
|
93
|
-
|
98
|
+
url << '?' << params.map { |k, v| "#{k}=#{v}" }.join('&') if params
|
99
|
+
stub_request(:get, url)
|
100
|
+
.to_return(body: f.read,
|
101
|
+
headers: { 'Content-Type' => 'application/json',
|
102
|
+
'Transfer-Encoding' => 'chunked' })
|
103
|
+
end.close
|
94
104
|
end
|
95
105
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kubernetes-objects
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
|
-
-
|
7
|
+
- Splunk Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-07-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -16,102 +16,116 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 1.9.1
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: 1.9.1
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: kubeclient
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 4.6.0
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: 4.6.0
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: http_parser.rb
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.5.3
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.5.3
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: bundler
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
44
58
|
requirements:
|
45
59
|
- - "~>"
|
46
60
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
61
|
+
version: '2.0'
|
48
62
|
type: :development
|
49
63
|
prerelease: false
|
50
64
|
version_requirements: !ruby/object:Gem::Requirement
|
51
65
|
requirements:
|
52
66
|
- - "~>"
|
53
67
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
68
|
+
version: '2.0'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
70
|
+
name: minitest
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
58
72
|
requirements:
|
59
73
|
- - "~>"
|
60
74
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
75
|
+
version: '5.11'
|
62
76
|
type: :development
|
63
77
|
prerelease: false
|
64
78
|
version_requirements: !ruby/object:Gem::Requirement
|
65
79
|
requirements:
|
66
80
|
- - "~>"
|
67
81
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
82
|
+
version: '5.11'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
84
|
+
name: rake
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
72
86
|
requirements:
|
73
|
-
- - "
|
87
|
+
- - ">="
|
74
88
|
- !ruby/object:Gem::Version
|
75
|
-
version: '
|
89
|
+
version: '12.0'
|
76
90
|
type: :development
|
77
91
|
prerelease: false
|
78
92
|
version_requirements: !ruby/object:Gem::Requirement
|
79
93
|
requirements:
|
80
|
-
- - "
|
94
|
+
- - ">="
|
81
95
|
- !ruby/object:Gem::Version
|
82
|
-
version: '
|
96
|
+
version: '12.0'
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
98
|
+
name: test-unit
|
85
99
|
requirement: !ruby/object:Gem::Requirement
|
86
100
|
requirements:
|
87
101
|
- - "~>"
|
88
102
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
103
|
+
version: '3.3'
|
90
104
|
type: :development
|
91
105
|
prerelease: false
|
92
106
|
version_requirements: !ruby/object:Gem::Requirement
|
93
107
|
requirements:
|
94
108
|
- - "~>"
|
95
109
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
110
|
+
version: '3.3'
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: webmock
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
100
114
|
requirements:
|
101
115
|
- - "~>"
|
102
116
|
- !ruby/object:Gem::Version
|
103
|
-
version: '3.
|
117
|
+
version: '3.5'
|
104
118
|
type: :development
|
105
119
|
prerelease: false
|
106
120
|
version_requirements: !ruby/object:Gem::Requirement
|
107
121
|
requirements:
|
108
122
|
- - "~>"
|
109
123
|
- !ruby/object:Gem::Version
|
110
|
-
version: '3.
|
124
|
+
version: '3.5'
|
111
125
|
description: A Fluentd input plugin for collecting Kubernetes objects, e.g. pods,
|
112
126
|
namespaces, events, etc. by pulling or watching.
|
113
127
|
email:
|
114
|
-
-
|
128
|
+
- DataEdge@splunk.com
|
115
129
|
executables: []
|
116
130
|
extensions: []
|
117
131
|
extra_rdoc_files: []
|
@@ -122,6 +136,7 @@ files:
|
|
122
136
|
- LICENSE
|
123
137
|
- README.md
|
124
138
|
- Rakefile
|
139
|
+
- VERSION
|
125
140
|
- fluent-plugin-kubernetes-objects.gemspec
|
126
141
|
- lib/fluent/plugin/in_kubernetes_objects.rb
|
127
142
|
- test/fluent/plugin/in_kubernetes_objects_test.rb
|
@@ -152,8 +167,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
152
167
|
- !ruby/object:Gem::Version
|
153
168
|
version: '0'
|
154
169
|
requirements: []
|
155
|
-
|
156
|
-
rubygems_version: 2.7.6
|
170
|
+
rubygems_version: 3.1.1
|
157
171
|
signing_key:
|
158
172
|
specification_version: 4
|
159
173
|
summary: Fluentd Plugin for Kubernetes Objects.
|