fluent-plugin-kubernetes_metadata_filter 2.1.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.circleci/config.yml +53 -0
- data/.gitignore +0 -2
- data/.rubocop.yml +57 -0
- data/Gemfile +4 -2
- data/Gemfile.lock +159 -0
- data/README.md +49 -60
- data/Rakefile +15 -11
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/readme.md +88 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/results.html +127 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/readme.md +97 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/results.html +136 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/readme.md +97 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/results.html +136 -0
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +25 -28
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +207 -187
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +30 -23
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +66 -24
- data/lib/fluent/plugin/kubernetes_metadata_stats.rb +21 -5
- data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
- data/lib/fluent/plugin/kubernetes_metadata_util.rb +33 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +154 -27
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +171 -29
- data/release_notes.md +42 -0
- data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -34
- data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -34
- data/test/cassettes/kubernetes_get_api_v1.yml +193 -0
- data/test/cassettes/kubernetes_get_api_v1_using_token.yml +195 -0
- data/test/cassettes/kubernetes_get_namespace_default.yml +72 -0
- data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +71 -0
- data/test/cassettes/{kubernetes_docker_metadata.yml → kubernetes_get_pod.yml} +0 -82
- data/test/cassettes/kubernetes_get_pod_container_init.yml +145 -0
- data/test/cassettes/{metadata_with_namespace_id.yml → kubernetes_get_pod_using_token.yml} +2 -130
- data/test/cassettes/{kubernetes_docker_metadata_using_bearer_token.yml → kubernetes_get_pod_with_ownerrefs.yml} +17 -109
- data/test/cassettes/metadata_from_tag_and_journald_fields.yml +153 -0
- data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +285 -0
- data/test/cassettes/{non_kubernetes_docker_metadata.yml → valid_kubernetes_api_server_using_token.yml} +4 -44
- data/test/helper.rb +20 -2
- data/test/plugin/test_cache_stats.rb +10 -13
- data/test/plugin/test_cache_strategy.rb +158 -160
- data/test/plugin/test_filter_kubernetes_metadata.rb +451 -314
- data/test/plugin/test_watch_namespaces.rb +209 -55
- data/test/plugin/test_watch_pods.rb +302 -71
- data/test/plugin/watch_test.rb +52 -33
- metadata +91 -70
- data/circle.yml +0 -17
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
|
|
2
|
+
<html>
|
|
3
|
+
<div>
|
|
4
|
+
<div><b>Options</b><div>
|
|
5
|
+
<div>Image: quay.io/openshift-logging/fluentd:1.14.6</div>
|
|
6
|
+
<div>Total Log Stressors: 1</div>
|
|
7
|
+
<div>Lines Per Second: 2500</div>
|
|
8
|
+
<div>Run Duration: 5m</div>
|
|
9
|
+
<div>Payload Source: synthetic</div>
|
|
10
|
+
</div>
|
|
11
|
+
<div>
|
|
12
|
+
Latency of logs collected based on the time the log was generated and ingested
|
|
13
|
+
</div>
|
|
14
|
+
<table border="1">
|
|
15
|
+
<tr>
|
|
16
|
+
<th>Total</th>
|
|
17
|
+
<th>Size</th>
|
|
18
|
+
<th>Elapsed</th>
|
|
19
|
+
<th>Mean</th>
|
|
20
|
+
<th>Min</th>
|
|
21
|
+
<th>Max</th>
|
|
22
|
+
<th>Median</th>
|
|
23
|
+
</tr>
|
|
24
|
+
<tr>
|
|
25
|
+
<th>Msg</th>
|
|
26
|
+
<th></th>
|
|
27
|
+
<th>(s)</th>
|
|
28
|
+
<th>(s)</th>
|
|
29
|
+
<th>(s)</th>
|
|
30
|
+
<th>(s)</th>
|
|
31
|
+
<th>(s)</th>
|
|
32
|
+
</tr>
|
|
33
|
+
<tr>
|
|
34
|
+
<td>705733</td>
|
|
35
|
+
<td>256</td>
|
|
36
|
+
<td>5m0s</td>
|
|
37
|
+
<td>16.929</td>
|
|
38
|
+
<td>1.826</td>
|
|
39
|
+
<td>36.399</td>
|
|
40
|
+
<td>14.956</td>
|
|
41
|
+
</tr>
|
|
42
|
+
</table>
|
|
43
|
+
<div>
|
|
44
|
+
<img src="cpu.png">
|
|
45
|
+
</div>
|
|
46
|
+
<div>
|
|
47
|
+
<img src="mem.png">
|
|
48
|
+
</div>
|
|
49
|
+
<div>
|
|
50
|
+
<img src="latency.png">
|
|
51
|
+
</div>
|
|
52
|
+
<div>
|
|
53
|
+
<img src="loss.png">
|
|
54
|
+
</div>
|
|
55
|
+
<div>
|
|
56
|
+
<table border="1">
|
|
57
|
+
<tr>
|
|
58
|
+
<th>Stream</th>
|
|
59
|
+
<th>Min Seq</th>
|
|
60
|
+
<th>Max Seq</th>
|
|
61
|
+
<th>Purged</th>
|
|
62
|
+
<th>Collected</th>
|
|
63
|
+
<th>Percent Collected</th>
|
|
64
|
+
</tr>
|
|
65
|
+
<tr>
|
|
66
|
+
<tr><td>functional.0.0000000000000000DBCED7977B8F643F</td><td>0</td><td>716850</td><td>0</td><td>705733</td><td>98.4%</td><tr>
|
|
67
|
+
</table>
|
|
68
|
+
</div>
|
|
69
|
+
<div>
|
|
70
|
+
<code style="display:block;white-space:pre-wrap">
|
|
71
|
+
|
|
72
|
+
<system>
|
|
73
|
+
log_level debug
|
|
74
|
+
</system>
|
|
75
|
+
|
|
76
|
+
<source>
|
|
77
|
+
@type tail
|
|
78
|
+
@id container-input
|
|
79
|
+
path /var/log/pods/testhack-clwn63sc_*/loader-*/*
|
|
80
|
+
exclude_path ["/var/log/pods/**/*/*.gz","/var/log/pods/**/*/*.tmp"]
|
|
81
|
+
pos_file "/var/lib/fluentd/pos/containers-app"
|
|
82
|
+
refresh_interval 5
|
|
83
|
+
rotate_wait 5
|
|
84
|
+
tag kubernetes.*
|
|
85
|
+
read_from_head "true"
|
|
86
|
+
<parse>
|
|
87
|
+
@type regexp
|
|
88
|
+
expression /^(?<@timestamp>[^\s]+) (?<stream>stdout|stderr) (?<logtag>[F|P]) (?<message>.*)$/
|
|
89
|
+
time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
|
|
90
|
+
keep_time_key true
|
|
91
|
+
</parse>
|
|
92
|
+
</source>
|
|
93
|
+
|
|
94
|
+
<filter kubernetes.**>
|
|
95
|
+
@type concat
|
|
96
|
+
key message
|
|
97
|
+
partial_key logtag
|
|
98
|
+
partial_value P
|
|
99
|
+
separator ''
|
|
100
|
+
</filter>
|
|
101
|
+
|
|
102
|
+
<match **>
|
|
103
|
+
@type forward
|
|
104
|
+
heartbeat_type none
|
|
105
|
+
keepalive true
|
|
106
|
+
|
|
107
|
+
<buffer>
|
|
108
|
+
flush_mode interval
|
|
109
|
+
flush_interval 5s
|
|
110
|
+
flush_at_shutdown true
|
|
111
|
+
flush_thread_count 2
|
|
112
|
+
retry_type exponential_backoff
|
|
113
|
+
retry_wait 1s
|
|
114
|
+
retry_max_interval 60s
|
|
115
|
+
retry_forever true
|
|
116
|
+
overflow_action block
|
|
117
|
+
</buffer>
|
|
118
|
+
|
|
119
|
+
<server>
|
|
120
|
+
host 0.0.0.0
|
|
121
|
+
port 24224
|
|
122
|
+
</server>
|
|
123
|
+
</match>
|
|
124
|
+
|
|
125
|
+
</code>
|
|
126
|
+
</div>
|
|
127
|
+
</html>
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
|
|
2
|
+
# collector Functionl Benchmark Results
|
|
3
|
+
## Options
|
|
4
|
+
* Image: quay.io/openshift-logging/fluentd:1.14.6
|
|
5
|
+
* Total Log Stressors: 1
|
|
6
|
+
* Lines Per Second: 2500
|
|
7
|
+
* Run Duration: 5m
|
|
8
|
+
* Payload Source: synthetic
|
|
9
|
+
|
|
10
|
+
## Latency of logs collected based on the time the log was generated and ingested
|
|
11
|
+
|
|
12
|
+
Total Msg| Size | Elapsed (s) | Mean (s)| Min(s) | Max (s)| Median (s)
|
|
13
|
+
---------|------|-------------|---------|--------|--------|---
|
|
14
|
+
606542|256|5m0s|32.779|1.295|75.390|31.641
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
## Percent logs lost between first and last collected sequence ids
|
|
25
|
+
Stream | Min Seq | Max Seq | Purged | Collected | Percent Collected |
|
|
26
|
+
-------| ---------| --------| -------|-----------|--------------|
|
|
27
|
+
| loader-0|0|606541|0|606542|100.0%
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
## Config
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
<system>
|
|
34
|
+
log_level debug
|
|
35
|
+
</system>
|
|
36
|
+
|
|
37
|
+
<source>
|
|
38
|
+
@type tail
|
|
39
|
+
@id container-input
|
|
40
|
+
path /var/log/pods/testhack-cyvquhgi_*/loader-*/*
|
|
41
|
+
exclude_path ["/var/log/pods/**/*/*.gz","/var/log/pods/**/*/*.tmp"]
|
|
42
|
+
pos_file "/var/lib/fluentd/pos/containers-app"
|
|
43
|
+
refresh_interval 5
|
|
44
|
+
rotate_wait 5
|
|
45
|
+
tag kubernetes.*
|
|
46
|
+
read_from_head "true"
|
|
47
|
+
<parse>
|
|
48
|
+
@type regexp
|
|
49
|
+
expression /^(?<@timestamp>[^\s]+) (?<stream>stdout|stderr) (?<logtag>[F|P]) (?<message>.*)$/
|
|
50
|
+
time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
|
|
51
|
+
keep_time_key true
|
|
52
|
+
</parse>
|
|
53
|
+
</source>
|
|
54
|
+
|
|
55
|
+
<filter kubernetes.**>
|
|
56
|
+
@type concat
|
|
57
|
+
key message
|
|
58
|
+
partial_key logtag
|
|
59
|
+
partial_value P
|
|
60
|
+
separator ''
|
|
61
|
+
</filter>
|
|
62
|
+
|
|
63
|
+
<filter kubernetes.**>
|
|
64
|
+
@id kubernetes-metadata
|
|
65
|
+
@type kubernetes_metadata
|
|
66
|
+
kubernetes_url 'https://kubernetes.default.svc'
|
|
67
|
+
annotation_match ["^containerType\.logging\.openshift\.io\/.*$"]
|
|
68
|
+
allow_orphans false
|
|
69
|
+
cache_size '1000'
|
|
70
|
+
use_journal 'nil'
|
|
71
|
+
ssl_partial_chain 'true'
|
|
72
|
+
</filter>
|
|
73
|
+
|
|
74
|
+
<match **>
|
|
75
|
+
@type forward
|
|
76
|
+
heartbeat_type none
|
|
77
|
+
keepalive true
|
|
78
|
+
|
|
79
|
+
<buffer>
|
|
80
|
+
flush_mode interval
|
|
81
|
+
flush_interval 5s
|
|
82
|
+
flush_at_shutdown true
|
|
83
|
+
flush_thread_count 2
|
|
84
|
+
retry_type exponential_backoff
|
|
85
|
+
retry_wait 1s
|
|
86
|
+
retry_max_interval 60s
|
|
87
|
+
retry_forever true
|
|
88
|
+
overflow_action block
|
|
89
|
+
</buffer>
|
|
90
|
+
|
|
91
|
+
<server>
|
|
92
|
+
host 0.0.0.0
|
|
93
|
+
port 24224
|
|
94
|
+
</server>
|
|
95
|
+
</match>
|
|
96
|
+
```
|
|
97
|
+
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
|
|
2
|
+
<html>
|
|
3
|
+
<div>
|
|
4
|
+
<div><b>Options</b><div>
|
|
5
|
+
<div>Image: quay.io/openshift-logging/fluentd:1.14.6</div>
|
|
6
|
+
<div>Total Log Stressors: 1</div>
|
|
7
|
+
<div>Lines Per Second: 2500</div>
|
|
8
|
+
<div>Run Duration: 5m</div>
|
|
9
|
+
<div>Payload Source: synthetic</div>
|
|
10
|
+
</div>
|
|
11
|
+
<div>
|
|
12
|
+
Latency of logs collected based on the time the log was generated and ingested
|
|
13
|
+
</div>
|
|
14
|
+
<table border="1">
|
|
15
|
+
<tr>
|
|
16
|
+
<th>Total</th>
|
|
17
|
+
<th>Size</th>
|
|
18
|
+
<th>Elapsed</th>
|
|
19
|
+
<th>Mean</th>
|
|
20
|
+
<th>Min</th>
|
|
21
|
+
<th>Max</th>
|
|
22
|
+
<th>Median</th>
|
|
23
|
+
</tr>
|
|
24
|
+
<tr>
|
|
25
|
+
<th>Msg</th>
|
|
26
|
+
<th></th>
|
|
27
|
+
<th>(s)</th>
|
|
28
|
+
<th>(s)</th>
|
|
29
|
+
<th>(s)</th>
|
|
30
|
+
<th>(s)</th>
|
|
31
|
+
<th>(s)</th>
|
|
32
|
+
</tr>
|
|
33
|
+
<tr>
|
|
34
|
+
<td>606542</td>
|
|
35
|
+
<td>256</td>
|
|
36
|
+
<td>5m0s</td>
|
|
37
|
+
<td>32.779</td>
|
|
38
|
+
<td>1.295</td>
|
|
39
|
+
<td>75.390</td>
|
|
40
|
+
<td>31.641</td>
|
|
41
|
+
</tr>
|
|
42
|
+
</table>
|
|
43
|
+
<div>
|
|
44
|
+
<img src="cpu.png">
|
|
45
|
+
</div>
|
|
46
|
+
<div>
|
|
47
|
+
<img src="mem.png">
|
|
48
|
+
</div>
|
|
49
|
+
<div>
|
|
50
|
+
<img src="latency.png">
|
|
51
|
+
</div>
|
|
52
|
+
<div>
|
|
53
|
+
<img src="loss.png">
|
|
54
|
+
</div>
|
|
55
|
+
<div>
|
|
56
|
+
<table border="1">
|
|
57
|
+
<tr>
|
|
58
|
+
<th>Stream</th>
|
|
59
|
+
<th>Min Seq</th>
|
|
60
|
+
<th>Max Seq</th>
|
|
61
|
+
<th>Purged</th>
|
|
62
|
+
<th>Collected</th>
|
|
63
|
+
<th>Percent Collected</th>
|
|
64
|
+
</tr>
|
|
65
|
+
<tr>
|
|
66
|
+
<tr><td>loader-0</td><td>0</td><td>606541</td><td>0</td><td>606542</td><td>100.0%</td><tr>
|
|
67
|
+
</table>
|
|
68
|
+
</div>
|
|
69
|
+
<div>
|
|
70
|
+
<code style="display:block;white-space:pre-wrap">
|
|
71
|
+
<system>
|
|
72
|
+
log_level debug
|
|
73
|
+
</system>
|
|
74
|
+
|
|
75
|
+
<source>
|
|
76
|
+
@type tail
|
|
77
|
+
@id container-input
|
|
78
|
+
path /var/log/pods/testhack-cyvquhgi_*/loader-*/*
|
|
79
|
+
exclude_path ["/var/log/pods/**/*/*.gz","/var/log/pods/**/*/*.tmp"]
|
|
80
|
+
pos_file "/var/lib/fluentd/pos/containers-app"
|
|
81
|
+
refresh_interval 5
|
|
82
|
+
rotate_wait 5
|
|
83
|
+
tag kubernetes.*
|
|
84
|
+
read_from_head "true"
|
|
85
|
+
<parse>
|
|
86
|
+
@type regexp
|
|
87
|
+
expression /^(?<@timestamp>[^\s]+) (?<stream>stdout|stderr) (?<logtag>[F|P]) (?<message>.*)$/
|
|
88
|
+
time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
|
|
89
|
+
keep_time_key true
|
|
90
|
+
</parse>
|
|
91
|
+
</source>
|
|
92
|
+
|
|
93
|
+
<filter kubernetes.**>
|
|
94
|
+
@type concat
|
|
95
|
+
key message
|
|
96
|
+
partial_key logtag
|
|
97
|
+
partial_value P
|
|
98
|
+
separator ''
|
|
99
|
+
</filter>
|
|
100
|
+
|
|
101
|
+
<filter kubernetes.**>
|
|
102
|
+
@id kubernetes-metadata
|
|
103
|
+
@type kubernetes_metadata
|
|
104
|
+
kubernetes_url 'https://kubernetes.default.svc'
|
|
105
|
+
annotation_match ["^containerType\.logging\.openshift\.io\/.*$"]
|
|
106
|
+
allow_orphans false
|
|
107
|
+
cache_size '1000'
|
|
108
|
+
use_journal 'nil'
|
|
109
|
+
ssl_partial_chain 'true'
|
|
110
|
+
</filter>
|
|
111
|
+
|
|
112
|
+
<match **>
|
|
113
|
+
@type forward
|
|
114
|
+
heartbeat_type none
|
|
115
|
+
keepalive true
|
|
116
|
+
|
|
117
|
+
<buffer>
|
|
118
|
+
flush_mode interval
|
|
119
|
+
flush_interval 5s
|
|
120
|
+
flush_at_shutdown true
|
|
121
|
+
flush_thread_count 2
|
|
122
|
+
retry_type exponential_backoff
|
|
123
|
+
retry_wait 1s
|
|
124
|
+
retry_max_interval 60s
|
|
125
|
+
retry_forever true
|
|
126
|
+
overflow_action block
|
|
127
|
+
</buffer>
|
|
128
|
+
|
|
129
|
+
<server>
|
|
130
|
+
host 0.0.0.0
|
|
131
|
+
port 24224
|
|
132
|
+
</server>
|
|
133
|
+
</match>
|
|
134
|
+
</code>
|
|
135
|
+
</div>
|
|
136
|
+
</html>
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
|
|
2
|
+
# collector Functionl Benchmark Results
|
|
3
|
+
## Options
|
|
4
|
+
* Image: quay.io/jcantril/fluentd:marshal
|
|
5
|
+
* Total Log Stressors: 1
|
|
6
|
+
* Lines Per Second: 2500
|
|
7
|
+
* Run Duration: 5m
|
|
8
|
+
* Payload Source: synthetic
|
|
9
|
+
|
|
10
|
+
## Latency of logs collected based on the time the log was generated and ingested
|
|
11
|
+
|
|
12
|
+
Total Msg| Size | Elapsed (s) | Mean (s)| Min(s) | Max (s)| Median (s)
|
|
13
|
+
---------|------|-------------|---------|--------|--------|---
|
|
14
|
+
722919|256|5m0s|14.322|1.077|32.992|13.513
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
## Percent logs lost between first and last collected sequence ids
|
|
25
|
+
Stream | Min Seq | Max Seq | Purged | Collected | Percent Collected |
|
|
26
|
+
-------| ---------| --------| -------|-----------|--------------|
|
|
27
|
+
| loader-0|0|722918|0|722919|100.0%
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
## Config
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
<system>
|
|
34
|
+
log_level debug
|
|
35
|
+
</system>
|
|
36
|
+
|
|
37
|
+
<source>
|
|
38
|
+
@type tail
|
|
39
|
+
@id container-input
|
|
40
|
+
path /var/log/pods/testhack-ksowqntt_*/loader-*/*
|
|
41
|
+
exclude_path ["/var/log/pods/**/*/*.gz","/var/log/pods/**/*/*.tmp"]
|
|
42
|
+
pos_file "/var/lib/fluentd/pos/containers-app"
|
|
43
|
+
refresh_interval 5
|
|
44
|
+
rotate_wait 5
|
|
45
|
+
tag kubernetes.*
|
|
46
|
+
read_from_head "true"
|
|
47
|
+
<parse>
|
|
48
|
+
@type regexp
|
|
49
|
+
expression /^(?<@timestamp>[^\s]+) (?<stream>stdout|stderr) (?<logtag>[F|P]) (?<message>.*)$/
|
|
50
|
+
time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
|
|
51
|
+
keep_time_key true
|
|
52
|
+
</parse>
|
|
53
|
+
</source>
|
|
54
|
+
|
|
55
|
+
<filter kubernetes.**>
|
|
56
|
+
@type concat
|
|
57
|
+
key message
|
|
58
|
+
partial_key logtag
|
|
59
|
+
partial_value P
|
|
60
|
+
separator ''
|
|
61
|
+
</filter>
|
|
62
|
+
|
|
63
|
+
<filter kubernetes.**>
|
|
64
|
+
@id kubernetes-metadata
|
|
65
|
+
@type kubernetes_metadata
|
|
66
|
+
kubernetes_url 'https://kubernetes.default.svc'
|
|
67
|
+
annotation_match ["^containerType\.logging\.openshift\.io\/.*$"]
|
|
68
|
+
allow_orphans false
|
|
69
|
+
cache_size '1000'
|
|
70
|
+
use_journal 'nil'
|
|
71
|
+
ssl_partial_chain 'true'
|
|
72
|
+
</filter>
|
|
73
|
+
|
|
74
|
+
<match **>
|
|
75
|
+
@type forward
|
|
76
|
+
heartbeat_type none
|
|
77
|
+
keepalive true
|
|
78
|
+
|
|
79
|
+
<buffer>
|
|
80
|
+
flush_mode interval
|
|
81
|
+
flush_interval 5s
|
|
82
|
+
flush_at_shutdown true
|
|
83
|
+
flush_thread_count 2
|
|
84
|
+
retry_type exponential_backoff
|
|
85
|
+
retry_wait 1s
|
|
86
|
+
retry_max_interval 60s
|
|
87
|
+
retry_forever true
|
|
88
|
+
overflow_action block
|
|
89
|
+
</buffer>
|
|
90
|
+
|
|
91
|
+
<server>
|
|
92
|
+
host 0.0.0.0
|
|
93
|
+
port 24224
|
|
94
|
+
</server>
|
|
95
|
+
</match>
|
|
96
|
+
```
|
|
97
|
+
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
|
|
2
|
+
<html>
|
|
3
|
+
<div>
|
|
4
|
+
<div><b>Options</b><div>
|
|
5
|
+
<div>Image: quay.io/jcantril/fluentd:marshal</div>
|
|
6
|
+
<div>Total Log Stressors: 1</div>
|
|
7
|
+
<div>Lines Per Second: 2500</div>
|
|
8
|
+
<div>Run Duration: 5m</div>
|
|
9
|
+
<div>Payload Source: synthetic</div>
|
|
10
|
+
</div>
|
|
11
|
+
<div>
|
|
12
|
+
Latency of logs collected based on the time the log was generated and ingested
|
|
13
|
+
</div>
|
|
14
|
+
<table border="1">
|
|
15
|
+
<tr>
|
|
16
|
+
<th>Total</th>
|
|
17
|
+
<th>Size</th>
|
|
18
|
+
<th>Elapsed</th>
|
|
19
|
+
<th>Mean</th>
|
|
20
|
+
<th>Min</th>
|
|
21
|
+
<th>Max</th>
|
|
22
|
+
<th>Median</th>
|
|
23
|
+
</tr>
|
|
24
|
+
<tr>
|
|
25
|
+
<th>Msg</th>
|
|
26
|
+
<th></th>
|
|
27
|
+
<th>(s)</th>
|
|
28
|
+
<th>(s)</th>
|
|
29
|
+
<th>(s)</th>
|
|
30
|
+
<th>(s)</th>
|
|
31
|
+
<th>(s)</th>
|
|
32
|
+
</tr>
|
|
33
|
+
<tr>
|
|
34
|
+
<td>722919</td>
|
|
35
|
+
<td>256</td>
|
|
36
|
+
<td>5m0s</td>
|
|
37
|
+
<td>14.322</td>
|
|
38
|
+
<td>1.077</td>
|
|
39
|
+
<td>32.992</td>
|
|
40
|
+
<td>13.513</td>
|
|
41
|
+
</tr>
|
|
42
|
+
</table>
|
|
43
|
+
<div>
|
|
44
|
+
<img src="cpu.png">
|
|
45
|
+
</div>
|
|
46
|
+
<div>
|
|
47
|
+
<img src="mem.png">
|
|
48
|
+
</div>
|
|
49
|
+
<div>
|
|
50
|
+
<img src="latency.png">
|
|
51
|
+
</div>
|
|
52
|
+
<div>
|
|
53
|
+
<img src="loss.png">
|
|
54
|
+
</div>
|
|
55
|
+
<div>
|
|
56
|
+
<table border="1">
|
|
57
|
+
<tr>
|
|
58
|
+
<th>Stream</th>
|
|
59
|
+
<th>Min Seq</th>
|
|
60
|
+
<th>Max Seq</th>
|
|
61
|
+
<th>Purged</th>
|
|
62
|
+
<th>Collected</th>
|
|
63
|
+
<th>Percent Collected</th>
|
|
64
|
+
</tr>
|
|
65
|
+
<tr>
|
|
66
|
+
<tr><td>loader-0</td><td>0</td><td>722918</td><td>0</td><td>722919</td><td>100.0%</td><tr>
|
|
67
|
+
</table>
|
|
68
|
+
</div>
|
|
69
|
+
<div>
|
|
70
|
+
<code style="display:block;white-space:pre-wrap">
|
|
71
|
+
<system>
|
|
72
|
+
log_level debug
|
|
73
|
+
</system>
|
|
74
|
+
|
|
75
|
+
<source>
|
|
76
|
+
@type tail
|
|
77
|
+
@id container-input
|
|
78
|
+
path /var/log/pods/testhack-ksowqntt_*/loader-*/*
|
|
79
|
+
exclude_path ["/var/log/pods/**/*/*.gz","/var/log/pods/**/*/*.tmp"]
|
|
80
|
+
pos_file "/var/lib/fluentd/pos/containers-app"
|
|
81
|
+
refresh_interval 5
|
|
82
|
+
rotate_wait 5
|
|
83
|
+
tag kubernetes.*
|
|
84
|
+
read_from_head "true"
|
|
85
|
+
<parse>
|
|
86
|
+
@type regexp
|
|
87
|
+
expression /^(?<@timestamp>[^\s]+) (?<stream>stdout|stderr) (?<logtag>[F|P]) (?<message>.*)$/
|
|
88
|
+
time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
|
|
89
|
+
keep_time_key true
|
|
90
|
+
</parse>
|
|
91
|
+
</source>
|
|
92
|
+
|
|
93
|
+
<filter kubernetes.**>
|
|
94
|
+
@type concat
|
|
95
|
+
key message
|
|
96
|
+
partial_key logtag
|
|
97
|
+
partial_value P
|
|
98
|
+
separator ''
|
|
99
|
+
</filter>
|
|
100
|
+
|
|
101
|
+
<filter kubernetes.**>
|
|
102
|
+
@id kubernetes-metadata
|
|
103
|
+
@type kubernetes_metadata
|
|
104
|
+
kubernetes_url 'https://kubernetes.default.svc'
|
|
105
|
+
annotation_match ["^containerType\.logging\.openshift\.io\/.*$"]
|
|
106
|
+
allow_orphans false
|
|
107
|
+
cache_size '1000'
|
|
108
|
+
use_journal 'nil'
|
|
109
|
+
ssl_partial_chain 'true'
|
|
110
|
+
</filter>
|
|
111
|
+
|
|
112
|
+
<match **>
|
|
113
|
+
@type forward
|
|
114
|
+
heartbeat_type none
|
|
115
|
+
keepalive true
|
|
116
|
+
|
|
117
|
+
<buffer>
|
|
118
|
+
flush_mode interval
|
|
119
|
+
flush_interval 5s
|
|
120
|
+
flush_at_shutdown true
|
|
121
|
+
flush_thread_count 2
|
|
122
|
+
retry_type exponential_backoff
|
|
123
|
+
retry_wait 1s
|
|
124
|
+
retry_max_interval 60s
|
|
125
|
+
retry_forever true
|
|
126
|
+
overflow_action block
|
|
127
|
+
</buffer>
|
|
128
|
+
|
|
129
|
+
<server>
|
|
130
|
+
host 0.0.0.0
|
|
131
|
+
port 24224
|
|
132
|
+
</server>
|
|
133
|
+
</match>
|
|
134
|
+
</code>
|
|
135
|
+
</div>
|
|
136
|
+
</html>
|
|
@@ -1,37 +1,34 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
5
|
|
|
5
6
|
Gem::Specification.new do |gem|
|
|
6
|
-
gem.name =
|
|
7
|
-
gem.version =
|
|
8
|
-
gem.authors = [
|
|
9
|
-
gem.email = [
|
|
10
|
-
gem.description =
|
|
11
|
-
gem.summary =
|
|
12
|
-
gem.homepage =
|
|
13
|
-
gem.license =
|
|
7
|
+
gem.name = 'fluent-plugin-kubernetes_metadata_filter'
|
|
8
|
+
gem.version = '3.4.0'
|
|
9
|
+
gem.authors = ['OpenShift Cluster Logging','Jimmi Dyson']
|
|
10
|
+
gem.email = ['team-logging@redhat.com','jimmidyson@gmail.com']
|
|
11
|
+
gem.description = 'Filter plugin to add Kubernetes metadata'
|
|
12
|
+
gem.summary = 'Fluentd filter plugin to add Kubernetes metadata'
|
|
13
|
+
gem.homepage = 'https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter'
|
|
14
|
+
gem.license = 'Apache-2.0'
|
|
14
15
|
|
|
15
16
|
gem.files = `git ls-files`.split($/)
|
|
16
|
-
gem.executables = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
|
17
|
-
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
|
18
|
-
gem.require_paths = ["lib"]
|
|
19
|
-
gem.has_rdoc = false
|
|
20
17
|
|
|
21
|
-
gem.required_ruby_version = '>= 2.
|
|
18
|
+
gem.required_ruby_version = '>= 2.6.0'
|
|
22
19
|
|
|
23
|
-
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '<
|
|
24
|
-
gem.add_runtime_dependency
|
|
25
|
-
gem.add_runtime_dependency
|
|
20
|
+
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.17']
|
|
21
|
+
gem.add_runtime_dependency 'kubeclient', ['>= 4.0.0', '< 5.0.0']
|
|
22
|
+
gem.add_runtime_dependency 'lru_redux'
|
|
26
23
|
|
|
27
|
-
gem.add_development_dependency
|
|
28
|
-
gem.add_development_dependency
|
|
29
|
-
gem.add_development_dependency
|
|
30
|
-
gem.add_development_dependency
|
|
31
|
-
gem.add_development_dependency
|
|
32
|
-
gem.add_development_dependency
|
|
33
|
-
gem.add_development_dependency
|
|
34
|
-
gem.add_development_dependency
|
|
35
|
-
gem.add_development_dependency
|
|
36
|
-
gem.add_development_dependency
|
|
24
|
+
gem.add_development_dependency 'bump'
|
|
25
|
+
gem.add_development_dependency 'bundler', '~> 2.0'
|
|
26
|
+
gem.add_development_dependency 'copyright-header'
|
|
27
|
+
gem.add_development_dependency 'minitest', '~> 4.0'
|
|
28
|
+
gem.add_development_dependency 'rake'
|
|
29
|
+
gem.add_development_dependency 'test-unit', '~> 3.5.5'
|
|
30
|
+
gem.add_development_dependency 'test-unit-rr', '~> 1.0.3'
|
|
31
|
+
gem.add_development_dependency 'vcr'
|
|
32
|
+
gem.add_development_dependency 'webmock'
|
|
33
|
+
gem.add_development_dependency 'yajl-ruby'
|
|
37
34
|
end
|