fluent-plugin-kubernetes_metadata_filter 3.3.0 → 3.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +4 -4
- data/README.md +1 -0
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +1 -1
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +1 -0
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +15 -2
- data/test/cassettes/kubernetes_get_pod_with_ownerrefs.yml +156 -0
- data/test/plugin/test_filter_kubernetes_metadata.rb +41 -0
- metadata +4 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0754e97a3a06ac6c24eee7004490eb0e06bfb51f63a2c973e07d9edfc593a239
|
|
4
|
+
data.tar.gz: 340c6bdc268bc2ff009fd63e58ec460e043f65636d1a03f9a3726467e84821f5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: de4fa7d241b0b7efacd1cde9e2c18c0bb3c8553d13e13528e9ccc16aa3ab974c12b1a88ce43a7bd186c9060d29ba649f8aa0365c3a9ecc4eed2c2354e1092dc9
|
|
7
|
+
data.tar.gz: 02ca1c7a7703e85962e206571b25a8416be206ef18d706958dfc894c598a068727e53abb3bd3beb059f1245211f957072b0b4c38a21a9d4c1ed114286306ffc9
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
fluent-plugin-kubernetes_metadata_filter (3.
|
|
4
|
+
fluent-plugin-kubernetes_metadata_filter (3.4.0)
|
|
5
5
|
fluentd (>= 0.14.0, < 1.17)
|
|
6
6
|
kubeclient (>= 4.0.0, < 5.0.0)
|
|
7
7
|
lru_redux
|
|
@@ -26,7 +26,7 @@ GEM
|
|
|
26
26
|
domain_name (0.5.20190701)
|
|
27
27
|
unf (>= 0.0.5, < 1.0.0)
|
|
28
28
|
escape_utils (1.2.2)
|
|
29
|
-
ffi (1.
|
|
29
|
+
ffi (1.16.2)
|
|
30
30
|
ffi-compiler (1.0.1)
|
|
31
31
|
ffi (>= 1.0.0)
|
|
32
32
|
rake
|
|
@@ -58,7 +58,7 @@ GEM
|
|
|
58
58
|
domain_name (~> 0.5)
|
|
59
59
|
http-form_data (2.3.0)
|
|
60
60
|
http_parser.rb (0.8.0)
|
|
61
|
-
jsonpath (1.1.
|
|
61
|
+
jsonpath (1.1.4)
|
|
62
62
|
multi_json
|
|
63
63
|
kubeclient (4.11.0)
|
|
64
64
|
http (>= 3.0, < 6.0)
|
|
@@ -69,7 +69,7 @@ GEM
|
|
|
69
69
|
ffi-compiler (~> 1.0)
|
|
70
70
|
rake (~> 13.0)
|
|
71
71
|
lru_redux (1.1.0)
|
|
72
|
-
mime-types (3.5.
|
|
72
|
+
mime-types (3.5.1)
|
|
73
73
|
mime-types-data (~> 3.2015)
|
|
74
74
|
mime-types-data (3.2023.0808)
|
|
75
75
|
mini_mime (1.1.2)
|
data/README.md
CHANGED
|
@@ -64,6 +64,7 @@ when true (default: `true`)
|
|
|
64
64
|
* `watch_retry_interval` - The time interval in seconds for retry backoffs when watch connections fail. (default: `10`)
|
|
65
65
|
* `open_timeout` - The time in seconds to wait for a connection to kubernetes service. (default: `3`)
|
|
66
66
|
* `read_timeout` - The time in seconds to wait for a read from kubernetes service. (default: `10`)
|
|
67
|
+
* `include_ownerrefs_metadata` - If set to true, it will include metadata (`kind` & `name`) in `kubernetes.ownerrefs` about the controller that owns the pod. (default: `false`)
|
|
67
68
|
|
|
68
69
|
|
|
69
70
|
Reading from a JSON formatted log files with `in_tail` and wildcard filenames while respecting the CRI-o log format with the same config you need the fluent-plugin "multi-format-parser":
|
|
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
5
5
|
|
|
6
6
|
Gem::Specification.new do |gem|
|
|
7
7
|
gem.name = 'fluent-plugin-kubernetes_metadata_filter'
|
|
8
|
-
gem.version = '3.
|
|
8
|
+
gem.version = '3.4.0'
|
|
9
9
|
gem.authors = ['OpenShift Cluster Logging','Jimmi Dyson']
|
|
10
10
|
gem.email = ['team-logging@redhat.com','jimmidyson@gmail.com']
|
|
11
11
|
gem.description = 'Filter plugin to add Kubernetes metadata'
|
|
@@ -82,6 +82,7 @@ module Fluent::Plugin
|
|
|
82
82
|
config_param :skip_container_metadata, :bool, default: false
|
|
83
83
|
config_param :skip_master_url, :bool, default: false
|
|
84
84
|
config_param :skip_namespace_metadata, :bool, default: false
|
|
85
|
+
config_param :include_ownerrefs_metadata, :bool, default: false
|
|
85
86
|
|
|
86
87
|
# A classname in the form of Test::APIAdapter which will try
|
|
87
88
|
# to be resolved from a relative named file 'test_api_adapter'
|
|
@@ -82,14 +82,27 @@ module KubernetesMetadata
|
|
|
82
82
|
log.warn("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
|
|
83
83
|
end
|
|
84
84
|
|
|
85
|
+
ownerrefs_meta = []
|
|
86
|
+
begin
|
|
87
|
+
pod_object[:metadata][:ownerReferences].each do |owner_reference|
|
|
88
|
+
ownerrefs_meta.append({
|
|
89
|
+
'kind' => owner_reference[:kind],
|
|
90
|
+
'name' => owner_reference[:name]
|
|
91
|
+
})
|
|
92
|
+
end
|
|
93
|
+
rescue StandardError => e
|
|
94
|
+
log.warn("parsing ownerrefs meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
|
|
95
|
+
end if @include_ownerrefs_metadata && pod_object[:metadata][:ownerReferences]
|
|
96
|
+
|
|
85
97
|
kubernetes_metadata = {
|
|
86
98
|
'namespace_name' => pod_object[:metadata][:namespace],
|
|
87
99
|
'pod_id' => pod_object[:metadata][:uid],
|
|
88
100
|
'pod_name' => pod_object[:metadata][:name],
|
|
89
101
|
'pod_ip' => pod_object[:status][:podIP],
|
|
90
102
|
'containers' => syms_to_strs(container_meta),
|
|
91
|
-
'host' => pod_object[:spec][:nodeName]
|
|
92
|
-
|
|
103
|
+
'host' => pod_object[:spec][:nodeName],
|
|
104
|
+
'ownerrefs' => (ownerrefs_meta if @include_ownerrefs_metadata)
|
|
105
|
+
}.compact
|
|
93
106
|
kubernetes_metadata['annotations'] = annotations unless annotations.empty?
|
|
94
107
|
kubernetes_metadata['labels'] = labels unless labels.empty?
|
|
95
108
|
kubernetes_metadata['master_url'] = @kubernetes_url unless @skip_master_url
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
|
3
|
+
# Kubernetes metadata
|
|
4
|
+
#
|
|
5
|
+
# Copyright 2015 Red Hat, Inc.
|
|
6
|
+
#
|
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
# you may not use this file except in compliance with the License.
|
|
9
|
+
# You may obtain a copy of the License at
|
|
10
|
+
#
|
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
#
|
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
# See the License for the specific language governing permissions and
|
|
17
|
+
# limitations under the License.
|
|
18
|
+
#
|
|
19
|
+
---
|
|
20
|
+
http_interactions:
|
|
21
|
+
- request:
|
|
22
|
+
method: get
|
|
23
|
+
uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
|
|
24
|
+
body:
|
|
25
|
+
encoding: US-ASCII
|
|
26
|
+
string: ''
|
|
27
|
+
headers:
|
|
28
|
+
Accept:
|
|
29
|
+
- "*/*; q=0.5, application/xml"
|
|
30
|
+
Accept-Encoding:
|
|
31
|
+
- gzip, deflate
|
|
32
|
+
User-Agent:
|
|
33
|
+
- Ruby
|
|
34
|
+
response:
|
|
35
|
+
status:
|
|
36
|
+
code: 200
|
|
37
|
+
message: OK
|
|
38
|
+
headers:
|
|
39
|
+
Content-Type:
|
|
40
|
+
- application/json
|
|
41
|
+
Date:
|
|
42
|
+
- Fri, 08 May 2015 10:35:37 GMT
|
|
43
|
+
Transfer-Encoding:
|
|
44
|
+
- chunked
|
|
45
|
+
body:
|
|
46
|
+
encoding: UTF-8
|
|
47
|
+
string: |-
|
|
48
|
+
{
|
|
49
|
+
"kind": "Pod",
|
|
50
|
+
"apiVersion": "v1",
|
|
51
|
+
"metadata": {
|
|
52
|
+
"name": "fabric8-console-controller-98rqc",
|
|
53
|
+
"generateName": "fabric8-console-controller-",
|
|
54
|
+
"namespace": "default",
|
|
55
|
+
"selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
|
|
56
|
+
"uid": "c76927af-f563-11e4-b32d-54ee7527188d",
|
|
57
|
+
"resourceVersion": "122",
|
|
58
|
+
"creationTimestamp": "2015-05-08T09:22:42Z",
|
|
59
|
+
"labels": {
|
|
60
|
+
"component": "fabric8Console"
|
|
61
|
+
},
|
|
62
|
+
"ownerReferences": [
|
|
63
|
+
{
|
|
64
|
+
"apiVersion": "apps/v1",
|
|
65
|
+
"blockOwnerDeletion": true,
|
|
66
|
+
"controller": true,
|
|
67
|
+
"kind": "ReplicaSet",
|
|
68
|
+
"name": "fabric8-console-controller",
|
|
69
|
+
"uid": "dd1ac3e1-bc88-4c54-be0f-26ac2406d884"
|
|
70
|
+
}
|
|
71
|
+
]
|
|
72
|
+
},
|
|
73
|
+
"spec": {
|
|
74
|
+
"volumes": [
|
|
75
|
+
{
|
|
76
|
+
"name": "openshift-cert-secrets",
|
|
77
|
+
"hostPath": null,
|
|
78
|
+
"emptyDir": null,
|
|
79
|
+
"gcePersistentDisk": null,
|
|
80
|
+
"gitRepo": null,
|
|
81
|
+
"secret": {
|
|
82
|
+
"secretName": "openshift-cert-secrets"
|
|
83
|
+
},
|
|
84
|
+
"nfs": null,
|
|
85
|
+
"iscsi": null,
|
|
86
|
+
"glusterfs": null
|
|
87
|
+
}
|
|
88
|
+
],
|
|
89
|
+
"containers": [
|
|
90
|
+
{
|
|
91
|
+
"name": "fabric8-console-container",
|
|
92
|
+
"image": "fabric8/hawtio-kubernetes:latest",
|
|
93
|
+
"ports": [
|
|
94
|
+
{
|
|
95
|
+
"containerPort": 9090,
|
|
96
|
+
"protocol": "TCP"
|
|
97
|
+
}
|
|
98
|
+
],
|
|
99
|
+
"env": [
|
|
100
|
+
{
|
|
101
|
+
"name": "OAUTH_CLIENT_ID",
|
|
102
|
+
"value": "fabric8-console"
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"name": "OAUTH_AUTHORIZE_URI",
|
|
106
|
+
"value": "https://localhost:8443/oauth/authorize"
|
|
107
|
+
}
|
|
108
|
+
],
|
|
109
|
+
"resources": {},
|
|
110
|
+
"volumeMounts": [
|
|
111
|
+
{
|
|
112
|
+
"name": "openshift-cert-secrets",
|
|
113
|
+
"readOnly": true,
|
|
114
|
+
"mountPath": "/etc/secret-volume"
|
|
115
|
+
}
|
|
116
|
+
],
|
|
117
|
+
"terminationMessagePath": "/dev/termination-log",
|
|
118
|
+
"imagePullPolicy": "IfNotPresent",
|
|
119
|
+
"capabilities": {}
|
|
120
|
+
}
|
|
121
|
+
],
|
|
122
|
+
"restartPolicy": "Always",
|
|
123
|
+
"dnsPolicy": "ClusterFirst",
|
|
124
|
+
"nodeName": "jimmi-redhat.localnet"
|
|
125
|
+
},
|
|
126
|
+
"status": {
|
|
127
|
+
"phase": "Running",
|
|
128
|
+
"Condition": [
|
|
129
|
+
{
|
|
130
|
+
"type": "Ready",
|
|
131
|
+
"status": "True"
|
|
132
|
+
}
|
|
133
|
+
],
|
|
134
|
+
"hostIP": "172.17.42.1",
|
|
135
|
+
"podIP": "172.17.0.8",
|
|
136
|
+
"containerStatuses": [
|
|
137
|
+
{
|
|
138
|
+
"name": "fabric8-console-container",
|
|
139
|
+
"state": {
|
|
140
|
+
"running": {
|
|
141
|
+
"startedAt": "2015-05-08T09:22:44Z"
|
|
142
|
+
}
|
|
143
|
+
},
|
|
144
|
+
"lastState": {},
|
|
145
|
+
"ready": true,
|
|
146
|
+
"restartCount": 0,
|
|
147
|
+
"image": "fabric8/hawtio-kubernetes:latest",
|
|
148
|
+
"imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
|
|
149
|
+
"containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459"
|
|
150
|
+
}
|
|
151
|
+
]
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
http_version:
|
|
155
|
+
recorded_at: Fri, 08 May 2015 10:35:37 GMT
|
|
156
|
+
recorded_with: VCR 2.9.3
|
|
@@ -805,6 +805,47 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
|
805
805
|
assert_equal(expected_kube_metadata, filtered[0])
|
|
806
806
|
end
|
|
807
807
|
end
|
|
808
|
+
test 'with docker & kubernetes metadata using include ownerrefs metadata' do
|
|
809
|
+
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' },
|
|
810
|
+
{ name: 'kubernetes_get_api_v1' },
|
|
811
|
+
{ name: 'kubernetes_get_pod_with_ownerrefs' },
|
|
812
|
+
{ name: 'kubernetes_get_namespace_default' }]) do
|
|
813
|
+
filtered = emit({}, '
|
|
814
|
+
kubernetes_url https://localhost:8443
|
|
815
|
+
watch false
|
|
816
|
+
cache_size 1
|
|
817
|
+
stats_interval 0
|
|
818
|
+
skip_pod_labels true
|
|
819
|
+
skip_master_url true
|
|
820
|
+
include_ownerrefs_metadata true
|
|
821
|
+
')
|
|
822
|
+
expected_kube_metadata = {
|
|
823
|
+
'docker'=>{
|
|
824
|
+
'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
|
825
|
+
},
|
|
826
|
+
'kubernetes' => {
|
|
827
|
+
'host' => 'jimmi-redhat.localnet',
|
|
828
|
+
'pod_name' => 'fabric8-console-controller-98rqc',
|
|
829
|
+
'container_name' => 'fabric8-console-container',
|
|
830
|
+
'container_image' => 'fabric8/hawtio-kubernetes:latest',
|
|
831
|
+
'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
|
|
832
|
+
'namespace_name' => 'default',
|
|
833
|
+
"namespace_id"=>"898268c8-4a36-11e5-9d81-42010af0194c",
|
|
834
|
+
'namespace_labels' => {
|
|
835
|
+
'tenant' => 'test'
|
|
836
|
+
},
|
|
837
|
+
'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
|
|
838
|
+
'pod_ip' => '172.17.0.8',
|
|
839
|
+
'ownerrefs' => [{
|
|
840
|
+
'kind' => 'ReplicaSet',
|
|
841
|
+
'name' => 'fabric8-console-controller'
|
|
842
|
+
}]
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
assert_equal(expected_kube_metadata, filtered[0])
|
|
847
|
+
end
|
|
848
|
+
end
|
|
808
849
|
|
|
809
850
|
end
|
|
810
851
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-kubernetes_metadata_filter
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 3.
|
|
4
|
+
version: 3.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- OpenShift Cluster Logging
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2023-
|
|
12
|
+
date: 2023-10-02 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: fluentd
|
|
@@ -259,6 +259,7 @@ files:
|
|
|
259
259
|
- test/cassettes/kubernetes_get_pod.yml
|
|
260
260
|
- test/cassettes/kubernetes_get_pod_container_init.yml
|
|
261
261
|
- test/cassettes/kubernetes_get_pod_using_token.yml
|
|
262
|
+
- test/cassettes/kubernetes_get_pod_with_ownerrefs.yml
|
|
262
263
|
- test/cassettes/metadata_from_tag_and_journald_fields.yml
|
|
263
264
|
- test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml
|
|
264
265
|
- test/cassettes/valid_kubernetes_api_server.yml
|
|
@@ -290,7 +291,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
290
291
|
- !ruby/object:Gem::Version
|
|
291
292
|
version: '0'
|
|
292
293
|
requirements: []
|
|
293
|
-
rubygems_version: 3.
|
|
294
|
+
rubygems_version: 3.3.3
|
|
294
295
|
signing_key:
|
|
295
296
|
specification_version: 4
|
|
296
297
|
summary: Fluentd filter plugin to add Kubernetes metadata
|