fluent-plugin-kubernetes_metadata_filter 2.1.0 → 3.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.circleci/config.yml +53 -0
- data/.gitignore +0 -2
- data/.rubocop.yml +57 -0
- data/Gemfile +4 -2
- data/Gemfile.lock +159 -0
- data/README.md +49 -60
- data/Rakefile +15 -11
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/readme.md +88 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/results.html +127 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/readme.md +97 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/results.html +136 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/readme.md +97 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/results.html +136 -0
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +25 -28
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +207 -187
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +30 -23
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +66 -24
- data/lib/fluent/plugin/kubernetes_metadata_stats.rb +21 -5
- data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
- data/lib/fluent/plugin/kubernetes_metadata_util.rb +33 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +154 -27
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +171 -29
- data/release_notes.md +42 -0
- data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -34
- data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -34
- data/test/cassettes/kubernetes_get_api_v1.yml +193 -0
- data/test/cassettes/kubernetes_get_api_v1_using_token.yml +195 -0
- data/test/cassettes/kubernetes_get_namespace_default.yml +72 -0
- data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +71 -0
- data/test/cassettes/{kubernetes_docker_metadata.yml → kubernetes_get_pod.yml} +0 -82
- data/test/cassettes/kubernetes_get_pod_container_init.yml +145 -0
- data/test/cassettes/{metadata_with_namespace_id.yml → kubernetes_get_pod_using_token.yml} +2 -130
- data/test/cassettes/{kubernetes_docker_metadata_using_bearer_token.yml → kubernetes_get_pod_with_ownerrefs.yml} +17 -109
- data/test/cassettes/metadata_from_tag_and_journald_fields.yml +153 -0
- data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +285 -0
- data/test/cassettes/{non_kubernetes_docker_metadata.yml → valid_kubernetes_api_server_using_token.yml} +4 -44
- data/test/helper.rb +20 -2
- data/test/plugin/test_cache_stats.rb +10 -13
- data/test/plugin/test_cache_strategy.rb +158 -160
- data/test/plugin/test_filter_kubernetes_metadata.rb +451 -314
- data/test/plugin/test_watch_namespaces.rb +209 -55
- data/test/plugin/test_watch_pods.rb +302 -71
- data/test/plugin/watch_test.rb +52 -33
- metadata +91 -70
- data/circle.yml +0 -17
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
|
|
2
|
+
<html>
|
|
3
|
+
<div>
|
|
4
|
+
<div><b>Options</b><div>
|
|
5
|
+
<div>Image: quay.io/openshift-logging/fluentd:1.14.6</div>
|
|
6
|
+
<div>Total Log Stressors: 1</div>
|
|
7
|
+
<div>Lines Per Second: 2500</div>
|
|
8
|
+
<div>Run Duration: 5m</div>
|
|
9
|
+
<div>Payload Source: synthetic</div>
|
|
10
|
+
</div>
|
|
11
|
+
<div>
|
|
12
|
+
Latency of logs collected based on the time the log was generated and ingested
|
|
13
|
+
</div>
|
|
14
|
+
<table border="1">
|
|
15
|
+
<tr>
|
|
16
|
+
<th>Total</th>
|
|
17
|
+
<th>Size</th>
|
|
18
|
+
<th>Elapsed</th>
|
|
19
|
+
<th>Mean</th>
|
|
20
|
+
<th>Min</th>
|
|
21
|
+
<th>Max</th>
|
|
22
|
+
<th>Median</th>
|
|
23
|
+
</tr>
|
|
24
|
+
<tr>
|
|
25
|
+
<th>Msg</th>
|
|
26
|
+
<th></th>
|
|
27
|
+
<th>(s)</th>
|
|
28
|
+
<th>(s)</th>
|
|
29
|
+
<th>(s)</th>
|
|
30
|
+
<th>(s)</th>
|
|
31
|
+
<th>(s)</th>
|
|
32
|
+
</tr>
|
|
33
|
+
<tr>
|
|
34
|
+
<td>705733</td>
|
|
35
|
+
<td>256</td>
|
|
36
|
+
<td>5m0s</td>
|
|
37
|
+
<td>16.929</td>
|
|
38
|
+
<td>1.826</td>
|
|
39
|
+
<td>36.399</td>
|
|
40
|
+
<td>14.956</td>
|
|
41
|
+
</tr>
|
|
42
|
+
</table>
|
|
43
|
+
<div>
|
|
44
|
+
<img src="cpu.png">
|
|
45
|
+
</div>
|
|
46
|
+
<div>
|
|
47
|
+
<img src="mem.png">
|
|
48
|
+
</div>
|
|
49
|
+
<div>
|
|
50
|
+
<img src="latency.png">
|
|
51
|
+
</div>
|
|
52
|
+
<div>
|
|
53
|
+
<img src="loss.png">
|
|
54
|
+
</div>
|
|
55
|
+
<div>
|
|
56
|
+
<table border="1">
|
|
57
|
+
<tr>
|
|
58
|
+
<th>Stream</th>
|
|
59
|
+
<th>Min Seq</th>
|
|
60
|
+
<th>Max Seq</th>
|
|
61
|
+
<th>Purged</th>
|
|
62
|
+
<th>Collected</th>
|
|
63
|
+
<th>Percent Collected</th>
|
|
64
|
+
</tr>
|
|
65
|
+
<tr>
|
|
66
|
+
<tr><td>functional.0.0000000000000000DBCED7977B8F643F</td><td>0</td><td>716850</td><td>0</td><td>705733</td><td>98.4%</td><tr>
|
|
67
|
+
</table>
|
|
68
|
+
</div>
|
|
69
|
+
<div>
|
|
70
|
+
<code style="display:block;white-space:pre-wrap">
|
|
71
|
+
|
|
72
|
+
<system>
|
|
73
|
+
log_level debug
|
|
74
|
+
</system>
|
|
75
|
+
|
|
76
|
+
<source>
|
|
77
|
+
@type tail
|
|
78
|
+
@id container-input
|
|
79
|
+
path /var/log/pods/testhack-clwn63sc_*/loader-*/*
|
|
80
|
+
exclude_path ["/var/log/pods/**/*/*.gz","/var/log/pods/**/*/*.tmp"]
|
|
81
|
+
pos_file "/var/lib/fluentd/pos/containers-app"
|
|
82
|
+
refresh_interval 5
|
|
83
|
+
rotate_wait 5
|
|
84
|
+
tag kubernetes.*
|
|
85
|
+
read_from_head "true"
|
|
86
|
+
<parse>
|
|
87
|
+
@type regexp
|
|
88
|
+
expression /^(?<@timestamp>[^\s]+) (?<stream>stdout|stderr) (?<logtag>[F|P]) (?<message>.*)$/
|
|
89
|
+
time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
|
|
90
|
+
keep_time_key true
|
|
91
|
+
</parse>
|
|
92
|
+
</source>
|
|
93
|
+
|
|
94
|
+
<filter kubernetes.**>
|
|
95
|
+
@type concat
|
|
96
|
+
key message
|
|
97
|
+
partial_key logtag
|
|
98
|
+
partial_value P
|
|
99
|
+
separator ''
|
|
100
|
+
</filter>
|
|
101
|
+
|
|
102
|
+
<match **>
|
|
103
|
+
@type forward
|
|
104
|
+
heartbeat_type none
|
|
105
|
+
keepalive true
|
|
106
|
+
|
|
107
|
+
<buffer>
|
|
108
|
+
flush_mode interval
|
|
109
|
+
flush_interval 5s
|
|
110
|
+
flush_at_shutdown true
|
|
111
|
+
flush_thread_count 2
|
|
112
|
+
retry_type exponential_backoff
|
|
113
|
+
retry_wait 1s
|
|
114
|
+
retry_max_interval 60s
|
|
115
|
+
retry_forever true
|
|
116
|
+
overflow_action block
|
|
117
|
+
</buffer>
|
|
118
|
+
|
|
119
|
+
<server>
|
|
120
|
+
host 0.0.0.0
|
|
121
|
+
port 24224
|
|
122
|
+
</server>
|
|
123
|
+
</match>
|
|
124
|
+
|
|
125
|
+
</code>
|
|
126
|
+
</div>
|
|
127
|
+
</html>
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
|
|
2
|
+
# collector Functionl Benchmark Results
|
|
3
|
+
## Options
|
|
4
|
+
* Image: quay.io/openshift-logging/fluentd:1.14.6
|
|
5
|
+
* Total Log Stressors: 1
|
|
6
|
+
* Lines Per Second: 2500
|
|
7
|
+
* Run Duration: 5m
|
|
8
|
+
* Payload Source: synthetic
|
|
9
|
+
|
|
10
|
+
## Latency of logs collected based on the time the log was generated and ingested
|
|
11
|
+
|
|
12
|
+
Total Msg| Size | Elapsed (s) | Mean (s)| Min(s) | Max (s)| Median (s)
|
|
13
|
+
---------|------|-------------|---------|--------|--------|---
|
|
14
|
+
606542|256|5m0s|32.779|1.295|75.390|31.641
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
## Percent logs lost between first and last collected sequence ids
|
|
25
|
+
Stream | Min Seq | Max Seq | Purged | Collected | Percent Collected |
|
|
26
|
+
-------| ---------| --------| -------|-----------|--------------|
|
|
27
|
+
| loader-0|0|606541|0|606542|100.0%
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
## Config
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
<system>
|
|
34
|
+
log_level debug
|
|
35
|
+
</system>
|
|
36
|
+
|
|
37
|
+
<source>
|
|
38
|
+
@type tail
|
|
39
|
+
@id container-input
|
|
40
|
+
path /var/log/pods/testhack-cyvquhgi_*/loader-*/*
|
|
41
|
+
exclude_path ["/var/log/pods/**/*/*.gz","/var/log/pods/**/*/*.tmp"]
|
|
42
|
+
pos_file "/var/lib/fluentd/pos/containers-app"
|
|
43
|
+
refresh_interval 5
|
|
44
|
+
rotate_wait 5
|
|
45
|
+
tag kubernetes.*
|
|
46
|
+
read_from_head "true"
|
|
47
|
+
<parse>
|
|
48
|
+
@type regexp
|
|
49
|
+
expression /^(?<@timestamp>[^\s]+) (?<stream>stdout|stderr) (?<logtag>[F|P]) (?<message>.*)$/
|
|
50
|
+
time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
|
|
51
|
+
keep_time_key true
|
|
52
|
+
</parse>
|
|
53
|
+
</source>
|
|
54
|
+
|
|
55
|
+
<filter kubernetes.**>
|
|
56
|
+
@type concat
|
|
57
|
+
key message
|
|
58
|
+
partial_key logtag
|
|
59
|
+
partial_value P
|
|
60
|
+
separator ''
|
|
61
|
+
</filter>
|
|
62
|
+
|
|
63
|
+
<filter kubernetes.**>
|
|
64
|
+
@id kubernetes-metadata
|
|
65
|
+
@type kubernetes_metadata
|
|
66
|
+
kubernetes_url 'https://kubernetes.default.svc'
|
|
67
|
+
annotation_match ["^containerType\.logging\.openshift\.io\/.*$"]
|
|
68
|
+
allow_orphans false
|
|
69
|
+
cache_size '1000'
|
|
70
|
+
use_journal 'nil'
|
|
71
|
+
ssl_partial_chain 'true'
|
|
72
|
+
</filter>
|
|
73
|
+
|
|
74
|
+
<match **>
|
|
75
|
+
@type forward
|
|
76
|
+
heartbeat_type none
|
|
77
|
+
keepalive true
|
|
78
|
+
|
|
79
|
+
<buffer>
|
|
80
|
+
flush_mode interval
|
|
81
|
+
flush_interval 5s
|
|
82
|
+
flush_at_shutdown true
|
|
83
|
+
flush_thread_count 2
|
|
84
|
+
retry_type exponential_backoff
|
|
85
|
+
retry_wait 1s
|
|
86
|
+
retry_max_interval 60s
|
|
87
|
+
retry_forever true
|
|
88
|
+
overflow_action block
|
|
89
|
+
</buffer>
|
|
90
|
+
|
|
91
|
+
<server>
|
|
92
|
+
host 0.0.0.0
|
|
93
|
+
port 24224
|
|
94
|
+
</server>
|
|
95
|
+
</match>
|
|
96
|
+
```
|
|
97
|
+
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
|
|
2
|
+
<html>
|
|
3
|
+
<div>
|
|
4
|
+
<div><b>Options</b><div>
|
|
5
|
+
<div>Image: quay.io/openshift-logging/fluentd:1.14.6</div>
|
|
6
|
+
<div>Total Log Stressors: 1</div>
|
|
7
|
+
<div>Lines Per Second: 2500</div>
|
|
8
|
+
<div>Run Duration: 5m</div>
|
|
9
|
+
<div>Payload Source: synthetic</div>
|
|
10
|
+
</div>
|
|
11
|
+
<div>
|
|
12
|
+
Latency of logs collected based on the time the log was generated and ingested
|
|
13
|
+
</div>
|
|
14
|
+
<table border="1">
|
|
15
|
+
<tr>
|
|
16
|
+
<th>Total</th>
|
|
17
|
+
<th>Size</th>
|
|
18
|
+
<th>Elapsed</th>
|
|
19
|
+
<th>Mean</th>
|
|
20
|
+
<th>Min</th>
|
|
21
|
+
<th>Max</th>
|
|
22
|
+
<th>Median</th>
|
|
23
|
+
</tr>
|
|
24
|
+
<tr>
|
|
25
|
+
<th>Msg</th>
|
|
26
|
+
<th></th>
|
|
27
|
+
<th>(s)</th>
|
|
28
|
+
<th>(s)</th>
|
|
29
|
+
<th>(s)</th>
|
|
30
|
+
<th>(s)</th>
|
|
31
|
+
<th>(s)</th>
|
|
32
|
+
</tr>
|
|
33
|
+
<tr>
|
|
34
|
+
<td>606542</td>
|
|
35
|
+
<td>256</td>
|
|
36
|
+
<td>5m0s</td>
|
|
37
|
+
<td>32.779</td>
|
|
38
|
+
<td>1.295</td>
|
|
39
|
+
<td>75.390</td>
|
|
40
|
+
<td>31.641</td>
|
|
41
|
+
</tr>
|
|
42
|
+
</table>
|
|
43
|
+
<div>
|
|
44
|
+
<img src="cpu.png">
|
|
45
|
+
</div>
|
|
46
|
+
<div>
|
|
47
|
+
<img src="mem.png">
|
|
48
|
+
</div>
|
|
49
|
+
<div>
|
|
50
|
+
<img src="latency.png">
|
|
51
|
+
</div>
|
|
52
|
+
<div>
|
|
53
|
+
<img src="loss.png">
|
|
54
|
+
</div>
|
|
55
|
+
<div>
|
|
56
|
+
<table border="1">
|
|
57
|
+
<tr>
|
|
58
|
+
<th>Stream</th>
|
|
59
|
+
<th>Min Seq</th>
|
|
60
|
+
<th>Max Seq</th>
|
|
61
|
+
<th>Purged</th>
|
|
62
|
+
<th>Collected</th>
|
|
63
|
+
<th>Percent Collected</th>
|
|
64
|
+
</tr>
|
|
65
|
+
<tr>
|
|
66
|
+
<tr><td>loader-0</td><td>0</td><td>606541</td><td>0</td><td>606542</td><td>100.0%</td><tr>
|
|
67
|
+
</table>
|
|
68
|
+
</div>
|
|
69
|
+
<div>
|
|
70
|
+
<code style="display:block;white-space:pre-wrap">
|
|
71
|
+
<system>
|
|
72
|
+
log_level debug
|
|
73
|
+
</system>
|
|
74
|
+
|
|
75
|
+
<source>
|
|
76
|
+
@type tail
|
|
77
|
+
@id container-input
|
|
78
|
+
path /var/log/pods/testhack-cyvquhgi_*/loader-*/*
|
|
79
|
+
exclude_path ["/var/log/pods/**/*/*.gz","/var/log/pods/**/*/*.tmp"]
|
|
80
|
+
pos_file "/var/lib/fluentd/pos/containers-app"
|
|
81
|
+
refresh_interval 5
|
|
82
|
+
rotate_wait 5
|
|
83
|
+
tag kubernetes.*
|
|
84
|
+
read_from_head "true"
|
|
85
|
+
<parse>
|
|
86
|
+
@type regexp
|
|
87
|
+
expression /^(?<@timestamp>[^\s]+) (?<stream>stdout|stderr) (?<logtag>[F|P]) (?<message>.*)$/
|
|
88
|
+
time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
|
|
89
|
+
keep_time_key true
|
|
90
|
+
</parse>
|
|
91
|
+
</source>
|
|
92
|
+
|
|
93
|
+
<filter kubernetes.**>
|
|
94
|
+
@type concat
|
|
95
|
+
key message
|
|
96
|
+
partial_key logtag
|
|
97
|
+
partial_value P
|
|
98
|
+
separator ''
|
|
99
|
+
</filter>
|
|
100
|
+
|
|
101
|
+
<filter kubernetes.**>
|
|
102
|
+
@id kubernetes-metadata
|
|
103
|
+
@type kubernetes_metadata
|
|
104
|
+
kubernetes_url 'https://kubernetes.default.svc'
|
|
105
|
+
annotation_match ["^containerType\.logging\.openshift\.io\/.*$"]
|
|
106
|
+
allow_orphans false
|
|
107
|
+
cache_size '1000'
|
|
108
|
+
use_journal 'nil'
|
|
109
|
+
ssl_partial_chain 'true'
|
|
110
|
+
</filter>
|
|
111
|
+
|
|
112
|
+
<match **>
|
|
113
|
+
@type forward
|
|
114
|
+
heartbeat_type none
|
|
115
|
+
keepalive true
|
|
116
|
+
|
|
117
|
+
<buffer>
|
|
118
|
+
flush_mode interval
|
|
119
|
+
flush_interval 5s
|
|
120
|
+
flush_at_shutdown true
|
|
121
|
+
flush_thread_count 2
|
|
122
|
+
retry_type exponential_backoff
|
|
123
|
+
retry_wait 1s
|
|
124
|
+
retry_max_interval 60s
|
|
125
|
+
retry_forever true
|
|
126
|
+
overflow_action block
|
|
127
|
+
</buffer>
|
|
128
|
+
|
|
129
|
+
<server>
|
|
130
|
+
host 0.0.0.0
|
|
131
|
+
port 24224
|
|
132
|
+
</server>
|
|
133
|
+
</match>
|
|
134
|
+
</code>
|
|
135
|
+
</div>
|
|
136
|
+
</html>
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
|
|
2
|
+
# collector Functionl Benchmark Results
|
|
3
|
+
## Options
|
|
4
|
+
* Image: quay.io/jcantril/fluentd:marshal
|
|
5
|
+
* Total Log Stressors: 1
|
|
6
|
+
* Lines Per Second: 2500
|
|
7
|
+
* Run Duration: 5m
|
|
8
|
+
* Payload Source: synthetic
|
|
9
|
+
|
|
10
|
+
## Latency of logs collected based on the time the log was generated and ingested
|
|
11
|
+
|
|
12
|
+
Total Msg| Size | Elapsed (s) | Mean (s)| Min(s) | Max (s)| Median (s)
|
|
13
|
+
---------|------|-------------|---------|--------|--------|---
|
|
14
|
+
722919|256|5m0s|14.322|1.077|32.992|13.513
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
## Percent logs lost between first and last collected sequence ids
|
|
25
|
+
Stream | Min Seq | Max Seq | Purged | Collected | Percent Collected |
|
|
26
|
+
-------| ---------| --------| -------|-----------|--------------|
|
|
27
|
+
| loader-0|0|722918|0|722919|100.0%
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
## Config
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
<system>
|
|
34
|
+
log_level debug
|
|
35
|
+
</system>
|
|
36
|
+
|
|
37
|
+
<source>
|
|
38
|
+
@type tail
|
|
39
|
+
@id container-input
|
|
40
|
+
path /var/log/pods/testhack-ksowqntt_*/loader-*/*
|
|
41
|
+
exclude_path ["/var/log/pods/**/*/*.gz","/var/log/pods/**/*/*.tmp"]
|
|
42
|
+
pos_file "/var/lib/fluentd/pos/containers-app"
|
|
43
|
+
refresh_interval 5
|
|
44
|
+
rotate_wait 5
|
|
45
|
+
tag kubernetes.*
|
|
46
|
+
read_from_head "true"
|
|
47
|
+
<parse>
|
|
48
|
+
@type regexp
|
|
49
|
+
expression /^(?<@timestamp>[^\s]+) (?<stream>stdout|stderr) (?<logtag>[F|P]) (?<message>.*)$/
|
|
50
|
+
time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
|
|
51
|
+
keep_time_key true
|
|
52
|
+
</parse>
|
|
53
|
+
</source>
|
|
54
|
+
|
|
55
|
+
<filter kubernetes.**>
|
|
56
|
+
@type concat
|
|
57
|
+
key message
|
|
58
|
+
partial_key logtag
|
|
59
|
+
partial_value P
|
|
60
|
+
separator ''
|
|
61
|
+
</filter>
|
|
62
|
+
|
|
63
|
+
<filter kubernetes.**>
|
|
64
|
+
@id kubernetes-metadata
|
|
65
|
+
@type kubernetes_metadata
|
|
66
|
+
kubernetes_url 'https://kubernetes.default.svc'
|
|
67
|
+
annotation_match ["^containerType\.logging\.openshift\.io\/.*$"]
|
|
68
|
+
allow_orphans false
|
|
69
|
+
cache_size '1000'
|
|
70
|
+
use_journal 'nil'
|
|
71
|
+
ssl_partial_chain 'true'
|
|
72
|
+
</filter>
|
|
73
|
+
|
|
74
|
+
<match **>
|
|
75
|
+
@type forward
|
|
76
|
+
heartbeat_type none
|
|
77
|
+
keepalive true
|
|
78
|
+
|
|
79
|
+
<buffer>
|
|
80
|
+
flush_mode interval
|
|
81
|
+
flush_interval 5s
|
|
82
|
+
flush_at_shutdown true
|
|
83
|
+
flush_thread_count 2
|
|
84
|
+
retry_type exponential_backoff
|
|
85
|
+
retry_wait 1s
|
|
86
|
+
retry_max_interval 60s
|
|
87
|
+
retry_forever true
|
|
88
|
+
overflow_action block
|
|
89
|
+
</buffer>
|
|
90
|
+
|
|
91
|
+
<server>
|
|
92
|
+
host 0.0.0.0
|
|
93
|
+
port 24224
|
|
94
|
+
</server>
|
|
95
|
+
</match>
|
|
96
|
+
```
|
|
97
|
+
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
|
|
2
|
+
<html>
|
|
3
|
+
<div>
|
|
4
|
+
<div><b>Options</b><div>
|
|
5
|
+
<div>Image: quay.io/jcantril/fluentd:marshal</div>
|
|
6
|
+
<div>Total Log Stressors: 1</div>
|
|
7
|
+
<div>Lines Per Second: 2500</div>
|
|
8
|
+
<div>Run Duration: 5m</div>
|
|
9
|
+
<div>Payload Source: synthetic</div>
|
|
10
|
+
</div>
|
|
11
|
+
<div>
|
|
12
|
+
Latency of logs collected based on the time the log was generated and ingested
|
|
13
|
+
</div>
|
|
14
|
+
<table border="1">
|
|
15
|
+
<tr>
|
|
16
|
+
<th>Total</th>
|
|
17
|
+
<th>Size</th>
|
|
18
|
+
<th>Elapsed</th>
|
|
19
|
+
<th>Mean</th>
|
|
20
|
+
<th>Min</th>
|
|
21
|
+
<th>Max</th>
|
|
22
|
+
<th>Median</th>
|
|
23
|
+
</tr>
|
|
24
|
+
<tr>
|
|
25
|
+
<th>Msg</th>
|
|
26
|
+
<th></th>
|
|
27
|
+
<th>(s)</th>
|
|
28
|
+
<th>(s)</th>
|
|
29
|
+
<th>(s)</th>
|
|
30
|
+
<th>(s)</th>
|
|
31
|
+
<th>(s)</th>
|
|
32
|
+
</tr>
|
|
33
|
+
<tr>
|
|
34
|
+
<td>722919</td>
|
|
35
|
+
<td>256</td>
|
|
36
|
+
<td>5m0s</td>
|
|
37
|
+
<td>14.322</td>
|
|
38
|
+
<td>1.077</td>
|
|
39
|
+
<td>32.992</td>
|
|
40
|
+
<td>13.513</td>
|
|
41
|
+
</tr>
|
|
42
|
+
</table>
|
|
43
|
+
<div>
|
|
44
|
+
<img src="cpu.png">
|
|
45
|
+
</div>
|
|
46
|
+
<div>
|
|
47
|
+
<img src="mem.png">
|
|
48
|
+
</div>
|
|
49
|
+
<div>
|
|
50
|
+
<img src="latency.png">
|
|
51
|
+
</div>
|
|
52
|
+
<div>
|
|
53
|
+
<img src="loss.png">
|
|
54
|
+
</div>
|
|
55
|
+
<div>
|
|
56
|
+
<table border="1">
|
|
57
|
+
<tr>
|
|
58
|
+
<th>Stream</th>
|
|
59
|
+
<th>Min Seq</th>
|
|
60
|
+
<th>Max Seq</th>
|
|
61
|
+
<th>Purged</th>
|
|
62
|
+
<th>Collected</th>
|
|
63
|
+
<th>Percent Collected</th>
|
|
64
|
+
</tr>
|
|
65
|
+
<tr>
|
|
66
|
+
<tr><td>loader-0</td><td>0</td><td>722918</td><td>0</td><td>722919</td><td>100.0%</td><tr>
|
|
67
|
+
</table>
|
|
68
|
+
</div>
|
|
69
|
+
<div>
|
|
70
|
+
<code style="display:block;white-space:pre-wrap">
|
|
71
|
+
<system>
|
|
72
|
+
log_level debug
|
|
73
|
+
</system>
|
|
74
|
+
|
|
75
|
+
<source>
|
|
76
|
+
@type tail
|
|
77
|
+
@id container-input
|
|
78
|
+
path /var/log/pods/testhack-ksowqntt_*/loader-*/*
|
|
79
|
+
exclude_path ["/var/log/pods/**/*/*.gz","/var/log/pods/**/*/*.tmp"]
|
|
80
|
+
pos_file "/var/lib/fluentd/pos/containers-app"
|
|
81
|
+
refresh_interval 5
|
|
82
|
+
rotate_wait 5
|
|
83
|
+
tag kubernetes.*
|
|
84
|
+
read_from_head "true"
|
|
85
|
+
<parse>
|
|
86
|
+
@type regexp
|
|
87
|
+
expression /^(?<@timestamp>[^\s]+) (?<stream>stdout|stderr) (?<logtag>[F|P]) (?<message>.*)$/
|
|
88
|
+
time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
|
|
89
|
+
keep_time_key true
|
|
90
|
+
</parse>
|
|
91
|
+
</source>
|
|
92
|
+
|
|
93
|
+
<filter kubernetes.**>
|
|
94
|
+
@type concat
|
|
95
|
+
key message
|
|
96
|
+
partial_key logtag
|
|
97
|
+
partial_value P
|
|
98
|
+
separator ''
|
|
99
|
+
</filter>
|
|
100
|
+
|
|
101
|
+
<filter kubernetes.**>
|
|
102
|
+
@id kubernetes-metadata
|
|
103
|
+
@type kubernetes_metadata
|
|
104
|
+
kubernetes_url 'https://kubernetes.default.svc'
|
|
105
|
+
annotation_match ["^containerType\.logging\.openshift\.io\/.*$"]
|
|
106
|
+
allow_orphans false
|
|
107
|
+
cache_size '1000'
|
|
108
|
+
use_journal 'nil'
|
|
109
|
+
ssl_partial_chain 'true'
|
|
110
|
+
</filter>
|
|
111
|
+
|
|
112
|
+
<match **>
|
|
113
|
+
@type forward
|
|
114
|
+
heartbeat_type none
|
|
115
|
+
keepalive true
|
|
116
|
+
|
|
117
|
+
<buffer>
|
|
118
|
+
flush_mode interval
|
|
119
|
+
flush_interval 5s
|
|
120
|
+
flush_at_shutdown true
|
|
121
|
+
flush_thread_count 2
|
|
122
|
+
retry_type exponential_backoff
|
|
123
|
+
retry_wait 1s
|
|
124
|
+
retry_max_interval 60s
|
|
125
|
+
retry_forever true
|
|
126
|
+
overflow_action block
|
|
127
|
+
</buffer>
|
|
128
|
+
|
|
129
|
+
<server>
|
|
130
|
+
host 0.0.0.0
|
|
131
|
+
port 24224
|
|
132
|
+
</server>
|
|
133
|
+
</match>
|
|
134
|
+
</code>
|
|
135
|
+
</div>
|
|
136
|
+
</html>
|
|
@@ -1,37 +1,34 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
5
|
|
|
5
6
|
Gem::Specification.new do |gem|
|
|
6
|
-
gem.name =
|
|
7
|
-
gem.version =
|
|
8
|
-
gem.authors = [
|
|
9
|
-
gem.email = [
|
|
10
|
-
gem.description =
|
|
11
|
-
gem.summary =
|
|
12
|
-
gem.homepage =
|
|
13
|
-
gem.license =
|
|
7
|
+
gem.name = 'fluent-plugin-kubernetes_metadata_filter'
|
|
8
|
+
gem.version = '3.4.0'
|
|
9
|
+
gem.authors = ['OpenShift Cluster Logging','Jimmi Dyson']
|
|
10
|
+
gem.email = ['team-logging@redhat.com','jimmidyson@gmail.com']
|
|
11
|
+
gem.description = 'Filter plugin to add Kubernetes metadata'
|
|
12
|
+
gem.summary = 'Fluentd filter plugin to add Kubernetes metadata'
|
|
13
|
+
gem.homepage = 'https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter'
|
|
14
|
+
gem.license = 'Apache-2.0'
|
|
14
15
|
|
|
15
16
|
gem.files = `git ls-files`.split($/)
|
|
16
|
-
gem.executables = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
|
17
|
-
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
|
18
|
-
gem.require_paths = ["lib"]
|
|
19
|
-
gem.has_rdoc = false
|
|
20
17
|
|
|
21
|
-
gem.required_ruby_version = '>= 2.
|
|
18
|
+
gem.required_ruby_version = '>= 2.6.0'
|
|
22
19
|
|
|
23
|
-
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '<
|
|
24
|
-
gem.add_runtime_dependency
|
|
25
|
-
gem.add_runtime_dependency
|
|
20
|
+
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.17']
|
|
21
|
+
gem.add_runtime_dependency 'kubeclient', ['>= 4.0.0', '< 5.0.0']
|
|
22
|
+
gem.add_runtime_dependency 'lru_redux'
|
|
26
23
|
|
|
27
|
-
gem.add_development_dependency
|
|
28
|
-
gem.add_development_dependency
|
|
29
|
-
gem.add_development_dependency
|
|
30
|
-
gem.add_development_dependency
|
|
31
|
-
gem.add_development_dependency
|
|
32
|
-
gem.add_development_dependency
|
|
33
|
-
gem.add_development_dependency
|
|
34
|
-
gem.add_development_dependency
|
|
35
|
-
gem.add_development_dependency
|
|
36
|
-
gem.add_development_dependency
|
|
24
|
+
gem.add_development_dependency 'bump'
|
|
25
|
+
gem.add_development_dependency 'bundler', '~> 2.0'
|
|
26
|
+
gem.add_development_dependency 'copyright-header'
|
|
27
|
+
gem.add_development_dependency 'minitest', '~> 4.0'
|
|
28
|
+
gem.add_development_dependency 'rake'
|
|
29
|
+
gem.add_development_dependency 'test-unit', '~> 3.5.5'
|
|
30
|
+
gem.add_development_dependency 'test-unit-rr', '~> 1.0.3'
|
|
31
|
+
gem.add_development_dependency 'vcr'
|
|
32
|
+
gem.add_development_dependency 'webmock'
|
|
33
|
+
gem.add_development_dependency 'yajl-ruby'
|
|
37
34
|
end
|