fluent-plugin-kubernetes_metadata_filter 3.3.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +4 -4
- data/README.md +1 -0
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +1 -1
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +1 -0
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +15 -2
- data/test/cassettes/kubernetes_get_pod_with_ownerrefs.yml +156 -0
- data/test/plugin/test_filter_kubernetes_metadata.rb +41 -0
- metadata +4 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0754e97a3a06ac6c24eee7004490eb0e06bfb51f63a2c973e07d9edfc593a239
|
|
4
|
+
data.tar.gz: 340c6bdc268bc2ff009fd63e58ec460e043f65636d1a03f9a3726467e84821f5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: de4fa7d241b0b7efacd1cde9e2c18c0bb3c8553d13e13528e9ccc16aa3ab974c12b1a88ce43a7bd186c9060d29ba649f8aa0365c3a9ecc4eed2c2354e1092dc9
|
|
7
|
+
data.tar.gz: 02ca1c7a7703e85962e206571b25a8416be206ef18d706958dfc894c598a068727e53abb3bd3beb059f1245211f957072b0b4c38a21a9d4c1ed114286306ffc9
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
fluent-plugin-kubernetes_metadata_filter (3.
|
|
4
|
+
fluent-plugin-kubernetes_metadata_filter (3.4.0)
|
|
5
5
|
fluentd (>= 0.14.0, < 1.17)
|
|
6
6
|
kubeclient (>= 4.0.0, < 5.0.0)
|
|
7
7
|
lru_redux
|
|
@@ -26,7 +26,7 @@ GEM
|
|
|
26
26
|
domain_name (0.5.20190701)
|
|
27
27
|
unf (>= 0.0.5, < 1.0.0)
|
|
28
28
|
escape_utils (1.2.2)
|
|
29
|
-
ffi (1.
|
|
29
|
+
ffi (1.16.2)
|
|
30
30
|
ffi-compiler (1.0.1)
|
|
31
31
|
ffi (>= 1.0.0)
|
|
32
32
|
rake
|
|
@@ -58,7 +58,7 @@ GEM
|
|
|
58
58
|
domain_name (~> 0.5)
|
|
59
59
|
http-form_data (2.3.0)
|
|
60
60
|
http_parser.rb (0.8.0)
|
|
61
|
-
jsonpath (1.1.
|
|
61
|
+
jsonpath (1.1.4)
|
|
62
62
|
multi_json
|
|
63
63
|
kubeclient (4.11.0)
|
|
64
64
|
http (>= 3.0, < 6.0)
|
|
@@ -69,7 +69,7 @@ GEM
|
|
|
69
69
|
ffi-compiler (~> 1.0)
|
|
70
70
|
rake (~> 13.0)
|
|
71
71
|
lru_redux (1.1.0)
|
|
72
|
-
mime-types (3.5.
|
|
72
|
+
mime-types (3.5.1)
|
|
73
73
|
mime-types-data (~> 3.2015)
|
|
74
74
|
mime-types-data (3.2023.0808)
|
|
75
75
|
mini_mime (1.1.2)
|
data/README.md
CHANGED
|
@@ -64,6 +64,7 @@ when true (default: `true`)
|
|
|
64
64
|
* `watch_retry_interval` - The time interval in seconds for retry backoffs when watch connections fail. (default: `10`)
|
|
65
65
|
* `open_timeout` - The time in seconds to wait for a connection to kubernetes service. (default: `3`)
|
|
66
66
|
* `read_timeout` - The time in seconds to wait for a read from kubernetes service. (default: `10`)
|
|
67
|
+
* `include_ownerrefs_metadata` - If set to true, it will include metadata (`kind` & `name`) in `kubernetes.ownerrefs` about the controller that owns the pod. (default: `false`)
|
|
67
68
|
|
|
68
69
|
|
|
69
70
|
Reading from a JSON formatted log files with `in_tail` and wildcard filenames while respecting the CRI-o log format with the same config you need the fluent-plugin "multi-format-parser":
|
|
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
5
5
|
|
|
6
6
|
Gem::Specification.new do |gem|
|
|
7
7
|
gem.name = 'fluent-plugin-kubernetes_metadata_filter'
|
|
8
|
-
gem.version = '3.
|
|
8
|
+
gem.version = '3.4.0'
|
|
9
9
|
gem.authors = ['OpenShift Cluster Logging','Jimmi Dyson']
|
|
10
10
|
gem.email = ['team-logging@redhat.com','jimmidyson@gmail.com']
|
|
11
11
|
gem.description = 'Filter plugin to add Kubernetes metadata'
|
|
@@ -82,6 +82,7 @@ module Fluent::Plugin
|
|
|
82
82
|
config_param :skip_container_metadata, :bool, default: false
|
|
83
83
|
config_param :skip_master_url, :bool, default: false
|
|
84
84
|
config_param :skip_namespace_metadata, :bool, default: false
|
|
85
|
+
config_param :include_ownerrefs_metadata, :bool, default: false
|
|
85
86
|
|
|
86
87
|
# A classname in the form of Test::APIAdapter which will try
|
|
87
88
|
# to be resolved from a relative named file 'test_api_adapter'
|
|
@@ -82,14 +82,27 @@ module KubernetesMetadata
|
|
|
82
82
|
log.warn("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
|
|
83
83
|
end
|
|
84
84
|
|
|
85
|
+
ownerrefs_meta = []
|
|
86
|
+
begin
|
|
87
|
+
pod_object[:metadata][:ownerReferences].each do |owner_reference|
|
|
88
|
+
ownerrefs_meta.append({
|
|
89
|
+
'kind' => owner_reference[:kind],
|
|
90
|
+
'name' => owner_reference[:name]
|
|
91
|
+
})
|
|
92
|
+
end
|
|
93
|
+
rescue StandardError => e
|
|
94
|
+
log.warn("parsing ownerrefs meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
|
|
95
|
+
end if @include_ownerrefs_metadata && pod_object[:metadata][:ownerReferences]
|
|
96
|
+
|
|
85
97
|
kubernetes_metadata = {
|
|
86
98
|
'namespace_name' => pod_object[:metadata][:namespace],
|
|
87
99
|
'pod_id' => pod_object[:metadata][:uid],
|
|
88
100
|
'pod_name' => pod_object[:metadata][:name],
|
|
89
101
|
'pod_ip' => pod_object[:status][:podIP],
|
|
90
102
|
'containers' => syms_to_strs(container_meta),
|
|
91
|
-
'host' => pod_object[:spec][:nodeName]
|
|
92
|
-
|
|
103
|
+
'host' => pod_object[:spec][:nodeName],
|
|
104
|
+
'ownerrefs' => (ownerrefs_meta if @include_ownerrefs_metadata)
|
|
105
|
+
}.compact
|
|
93
106
|
kubernetes_metadata['annotations'] = annotations unless annotations.empty?
|
|
94
107
|
kubernetes_metadata['labels'] = labels unless labels.empty?
|
|
95
108
|
kubernetes_metadata['master_url'] = @kubernetes_url unless @skip_master_url
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
|
3
|
+
# Kubernetes metadata
|
|
4
|
+
#
|
|
5
|
+
# Copyright 2015 Red Hat, Inc.
|
|
6
|
+
#
|
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
# you may not use this file except in compliance with the License.
|
|
9
|
+
# You may obtain a copy of the License at
|
|
10
|
+
#
|
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
#
|
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
# See the License for the specific language governing permissions and
|
|
17
|
+
# limitations under the License.
|
|
18
|
+
#
|
|
19
|
+
---
|
|
20
|
+
http_interactions:
|
|
21
|
+
- request:
|
|
22
|
+
method: get
|
|
23
|
+
uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
|
|
24
|
+
body:
|
|
25
|
+
encoding: US-ASCII
|
|
26
|
+
string: ''
|
|
27
|
+
headers:
|
|
28
|
+
Accept:
|
|
29
|
+
- "*/*; q=0.5, application/xml"
|
|
30
|
+
Accept-Encoding:
|
|
31
|
+
- gzip, deflate
|
|
32
|
+
User-Agent:
|
|
33
|
+
- Ruby
|
|
34
|
+
response:
|
|
35
|
+
status:
|
|
36
|
+
code: 200
|
|
37
|
+
message: OK
|
|
38
|
+
headers:
|
|
39
|
+
Content-Type:
|
|
40
|
+
- application/json
|
|
41
|
+
Date:
|
|
42
|
+
- Fri, 08 May 2015 10:35:37 GMT
|
|
43
|
+
Transfer-Encoding:
|
|
44
|
+
- chunked
|
|
45
|
+
body:
|
|
46
|
+
encoding: UTF-8
|
|
47
|
+
string: |-
|
|
48
|
+
{
|
|
49
|
+
"kind": "Pod",
|
|
50
|
+
"apiVersion": "v1",
|
|
51
|
+
"metadata": {
|
|
52
|
+
"name": "fabric8-console-controller-98rqc",
|
|
53
|
+
"generateName": "fabric8-console-controller-",
|
|
54
|
+
"namespace": "default",
|
|
55
|
+
"selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
|
|
56
|
+
"uid": "c76927af-f563-11e4-b32d-54ee7527188d",
|
|
57
|
+
"resourceVersion": "122",
|
|
58
|
+
"creationTimestamp": "2015-05-08T09:22:42Z",
|
|
59
|
+
"labels": {
|
|
60
|
+
"component": "fabric8Console"
|
|
61
|
+
},
|
|
62
|
+
"ownerReferences": [
|
|
63
|
+
{
|
|
64
|
+
"apiVersion": "apps/v1",
|
|
65
|
+
"blockOwnerDeletion": true,
|
|
66
|
+
"controller": true,
|
|
67
|
+
"kind": "ReplicaSet",
|
|
68
|
+
"name": "fabric8-console-controller",
|
|
69
|
+
"uid": "dd1ac3e1-bc88-4c54-be0f-26ac2406d884"
|
|
70
|
+
}
|
|
71
|
+
]
|
|
72
|
+
},
|
|
73
|
+
"spec": {
|
|
74
|
+
"volumes": [
|
|
75
|
+
{
|
|
76
|
+
"name": "openshift-cert-secrets",
|
|
77
|
+
"hostPath": null,
|
|
78
|
+
"emptyDir": null,
|
|
79
|
+
"gcePersistentDisk": null,
|
|
80
|
+
"gitRepo": null,
|
|
81
|
+
"secret": {
|
|
82
|
+
"secretName": "openshift-cert-secrets"
|
|
83
|
+
},
|
|
84
|
+
"nfs": null,
|
|
85
|
+
"iscsi": null,
|
|
86
|
+
"glusterfs": null
|
|
87
|
+
}
|
|
88
|
+
],
|
|
89
|
+
"containers": [
|
|
90
|
+
{
|
|
91
|
+
"name": "fabric8-console-container",
|
|
92
|
+
"image": "fabric8/hawtio-kubernetes:latest",
|
|
93
|
+
"ports": [
|
|
94
|
+
{
|
|
95
|
+
"containerPort": 9090,
|
|
96
|
+
"protocol": "TCP"
|
|
97
|
+
}
|
|
98
|
+
],
|
|
99
|
+
"env": [
|
|
100
|
+
{
|
|
101
|
+
"name": "OAUTH_CLIENT_ID",
|
|
102
|
+
"value": "fabric8-console"
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"name": "OAUTH_AUTHORIZE_URI",
|
|
106
|
+
"value": "https://localhost:8443/oauth/authorize"
|
|
107
|
+
}
|
|
108
|
+
],
|
|
109
|
+
"resources": {},
|
|
110
|
+
"volumeMounts": [
|
|
111
|
+
{
|
|
112
|
+
"name": "openshift-cert-secrets",
|
|
113
|
+
"readOnly": true,
|
|
114
|
+
"mountPath": "/etc/secret-volume"
|
|
115
|
+
}
|
|
116
|
+
],
|
|
117
|
+
"terminationMessagePath": "/dev/termination-log",
|
|
118
|
+
"imagePullPolicy": "IfNotPresent",
|
|
119
|
+
"capabilities": {}
|
|
120
|
+
}
|
|
121
|
+
],
|
|
122
|
+
"restartPolicy": "Always",
|
|
123
|
+
"dnsPolicy": "ClusterFirst",
|
|
124
|
+
"nodeName": "jimmi-redhat.localnet"
|
|
125
|
+
},
|
|
126
|
+
"status": {
|
|
127
|
+
"phase": "Running",
|
|
128
|
+
"Condition": [
|
|
129
|
+
{
|
|
130
|
+
"type": "Ready",
|
|
131
|
+
"status": "True"
|
|
132
|
+
}
|
|
133
|
+
],
|
|
134
|
+
"hostIP": "172.17.42.1",
|
|
135
|
+
"podIP": "172.17.0.8",
|
|
136
|
+
"containerStatuses": [
|
|
137
|
+
{
|
|
138
|
+
"name": "fabric8-console-container",
|
|
139
|
+
"state": {
|
|
140
|
+
"running": {
|
|
141
|
+
"startedAt": "2015-05-08T09:22:44Z"
|
|
142
|
+
}
|
|
143
|
+
},
|
|
144
|
+
"lastState": {},
|
|
145
|
+
"ready": true,
|
|
146
|
+
"restartCount": 0,
|
|
147
|
+
"image": "fabric8/hawtio-kubernetes:latest",
|
|
148
|
+
"imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
|
|
149
|
+
"containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459"
|
|
150
|
+
}
|
|
151
|
+
]
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
http_version:
|
|
155
|
+
recorded_at: Fri, 08 May 2015 10:35:37 GMT
|
|
156
|
+
recorded_with: VCR 2.9.3
|
|
@@ -805,6 +805,47 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
|
805
805
|
assert_equal(expected_kube_metadata, filtered[0])
|
|
806
806
|
end
|
|
807
807
|
end
|
|
808
|
+
test 'with docker & kubernetes metadata using include ownerrefs metadata' do
|
|
809
|
+
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' },
|
|
810
|
+
{ name: 'kubernetes_get_api_v1' },
|
|
811
|
+
{ name: 'kubernetes_get_pod_with_ownerrefs' },
|
|
812
|
+
{ name: 'kubernetes_get_namespace_default' }]) do
|
|
813
|
+
filtered = emit({}, '
|
|
814
|
+
kubernetes_url https://localhost:8443
|
|
815
|
+
watch false
|
|
816
|
+
cache_size 1
|
|
817
|
+
stats_interval 0
|
|
818
|
+
skip_pod_labels true
|
|
819
|
+
skip_master_url true
|
|
820
|
+
include_ownerrefs_metadata true
|
|
821
|
+
')
|
|
822
|
+
expected_kube_metadata = {
|
|
823
|
+
'docker'=>{
|
|
824
|
+
'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
|
825
|
+
},
|
|
826
|
+
'kubernetes' => {
|
|
827
|
+
'host' => 'jimmi-redhat.localnet',
|
|
828
|
+
'pod_name' => 'fabric8-console-controller-98rqc',
|
|
829
|
+
'container_name' => 'fabric8-console-container',
|
|
830
|
+
'container_image' => 'fabric8/hawtio-kubernetes:latest',
|
|
831
|
+
'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
|
|
832
|
+
'namespace_name' => 'default',
|
|
833
|
+
"namespace_id"=>"898268c8-4a36-11e5-9d81-42010af0194c",
|
|
834
|
+
'namespace_labels' => {
|
|
835
|
+
'tenant' => 'test'
|
|
836
|
+
},
|
|
837
|
+
'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
|
|
838
|
+
'pod_ip' => '172.17.0.8',
|
|
839
|
+
'ownerrefs' => [{
|
|
840
|
+
'kind' => 'ReplicaSet',
|
|
841
|
+
'name' => 'fabric8-console-controller'
|
|
842
|
+
}]
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
assert_equal(expected_kube_metadata, filtered[0])
|
|
847
|
+
end
|
|
848
|
+
end
|
|
808
849
|
|
|
809
850
|
end
|
|
810
851
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-kubernetes_metadata_filter
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 3.
|
|
4
|
+
version: 3.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- OpenShift Cluster Logging
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2023-
|
|
12
|
+
date: 2023-10-02 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: fluentd
|
|
@@ -259,6 +259,7 @@ files:
|
|
|
259
259
|
- test/cassettes/kubernetes_get_pod.yml
|
|
260
260
|
- test/cassettes/kubernetes_get_pod_container_init.yml
|
|
261
261
|
- test/cassettes/kubernetes_get_pod_using_token.yml
|
|
262
|
+
- test/cassettes/kubernetes_get_pod_with_ownerrefs.yml
|
|
262
263
|
- test/cassettes/metadata_from_tag_and_journald_fields.yml
|
|
263
264
|
- test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml
|
|
264
265
|
- test/cassettes/valid_kubernetes_api_server.yml
|
|
@@ -290,7 +291,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
290
291
|
- !ruby/object:Gem::Version
|
|
291
292
|
version: '0'
|
|
292
293
|
requirements: []
|
|
293
|
-
rubygems_version: 3.
|
|
294
|
+
rubygems_version: 3.3.3
|
|
294
295
|
signing_key:
|
|
295
296
|
specification_version: 4
|
|
296
297
|
summary: Fluentd filter plugin to add Kubernetes metadata
|