fluent-plugin-kubernetes_metadata_filter 2.13.0 → 3.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +13 -55
  3. data/Rakefile +3 -4
  4. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/cpu.png +0 -0
  5. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/latency.png +0 -0
  6. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/loss.png +0 -0
  7. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/mem.png +0 -0
  8. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/readme.md +88 -0
  9. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/results.html +127 -0
  10. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/cpu.png +0 -0
  11. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/latency.png +0 -0
  12. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/loss.png +0 -0
  13. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/mem.png +0 -0
  14. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/readme.md +97 -0
  15. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/results.html +136 -0
  16. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/cpu.png +0 -0
  17. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/latency.png +0 -0
  18. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/loss.png +0 -0
  19. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/mem.png +0 -0
  20. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/readme.md +97 -0
  21. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/results.html +136 -0
  22. data/lib/fluent/plugin/filter_kubernetes_metadata.rb +308 -330
  23. data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +15 -11
  24. data/lib/fluent/plugin/kubernetes_metadata_common.rb +45 -49
  25. data/lib/fluent/plugin/kubernetes_metadata_stats.rb +18 -4
  26. data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +39 -36
  27. data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +15 -12
  28. data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +19 -18
  29. metadata +29 -182
  30. data/.circleci/config.yml +0 -53
  31. data/.gitignore +0 -19
  32. data/.rubocop.yml +0 -57
  33. data/Gemfile +0 -9
  34. data/Gemfile.lock +0 -158
  35. data/fluent-plugin-kubernetes_metadata_filter.gemspec +0 -34
  36. data/lib/fluent/plugin/kubernetes_metadata_util.rb +0 -53
  37. data/test/cassettes/invalid_api_server_config.yml +0 -53
  38. data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -205
  39. data/test/cassettes/kubernetes_docker_metadata_dotted_slashed_labels.yml +0 -197
  40. data/test/cassettes/kubernetes_get_api_v1.yml +0 -193
  41. data/test/cassettes/kubernetes_get_api_v1_using_token.yml +0 -195
  42. data/test/cassettes/kubernetes_get_namespace_default.yml +0 -69
  43. data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +0 -71
  44. data/test/cassettes/kubernetes_get_pod.yml +0 -146
  45. data/test/cassettes/kubernetes_get_pod_container_init.yml +0 -145
  46. data/test/cassettes/kubernetes_get_pod_using_token.yml +0 -148
  47. data/test/cassettes/metadata_from_tag_and_journald_fields.yml +0 -153
  48. data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +0 -285
  49. data/test/cassettes/valid_kubernetes_api_server.yml +0 -55
  50. data/test/cassettes/valid_kubernetes_api_server_using_token.yml +0 -57
  51. data/test/helper.rb +0 -82
  52. data/test/plugin/test.token +0 -1
  53. data/test/plugin/test_cache_stats.rb +0 -33
  54. data/test/plugin/test_cache_strategy.rb +0 -194
  55. data/test/plugin/test_filter_kubernetes_metadata.rb +0 -1093
  56. data/test/plugin/test_utils.rb +0 -56
  57. data/test/plugin/test_watch_namespaces.rb +0 -245
  58. data/test/plugin/test_watch_pods.rb +0 -344
  59. data/test/plugin/watch_test.rb +0 -76
data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/readme.md ADDED
@@ -0,0 +1,97 @@
1
+
2
+ # collector Functionl Benchmark Results
3
+ ## Options
4
+ * Image: quay.io/jcantril/fluentd:marshal
5
+ * Total Log Stressors: 1
6
+ * Lines Per Second: 2500
7
+ * Run Duration: 5m
8
+ * Payload Source: synthetic
9
+
10
+ ## Latency of logs collected based on the time the log was generated and ingested
11
+
12
+ Total Msg| Size | Elapsed (s) | Mean (s)| Min(s) | Max (s)| Median (s)
13
+ ---------|------|-------------|---------|--------|--------|---
14
+ 722919|256|5m0s|14.322|1.077|32.992|13.513
15
+
16
+ ![](cpu.png)
17
+
18
+ ![](mem.png)
19
+
20
+ ![](latency.png)
21
+
22
+ ![](loss.png)
23
+
24
+ ## Percent logs lost between first and last collected sequence ids
25
+ Stream | Min Seq | Max Seq | Purged | Collected | Percent Collected |
26
+ -------| ---------| --------| -------|-----------|--------------|
27
+ | loader-0|0|722918|0|722919|100.0%
28
+
29
+
30
+ ## Config
31
+
32
+ ```
33
+ <system>
34
+ log_level debug
35
+ </system>
36
+
37
+ <source>
38
+ @type tail
39
+ @id container-input
40
+ path /var/log/pods/testhack-ksowqntt_*/loader-*/*
41
+ exclude_path ["/var/log/pods/**/*/*.gz","/var/log/pods/**/*/*.tmp"]
42
+ pos_file "/var/lib/fluentd/pos/containers-app"
43
+ refresh_interval 5
44
+ rotate_wait 5
45
+ tag kubernetes.*
46
+ read_from_head "true"
47
+ <parse>
48
+ @type regexp
49
+ expression /^(?<@timestamp>[^\s]+) (?<stream>stdout|stderr) (?<logtag>[F|P]) (?<message>.*)$/
50
+ time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
51
+ keep_time_key true
52
+ </parse>
53
+ </source>
54
+
55
+ <filter kubernetes.**>
56
+ @type concat
57
+ key message
58
+ partial_key logtag
59
+ partial_value P
60
+ separator ''
61
+ </filter>
62
+
63
+ <filter kubernetes.**>
64
+ @id kubernetes-metadata
65
+ @type kubernetes_metadata
66
+ kubernetes_url 'https://kubernetes.default.svc'
67
+ annotation_match ["^containerType\.logging\.openshift\.io\/.*$"]
68
+ allow_orphans false
69
+ cache_size '1000'
70
+ use_journal 'nil'
71
+ ssl_partial_chain 'true'
72
+ </filter>
73
+
74
+ <match **>
75
+ @type forward
76
+ heartbeat_type none
77
+ keepalive true
78
+
79
+ <buffer>
80
+ flush_mode interval
81
+ flush_interval 5s
82
+ flush_at_shutdown true
83
+ flush_thread_count 2
84
+ retry_type exponential_backoff
85
+ retry_wait 1s
86
+ retry_max_interval 60s
87
+ retry_forever true
88
+ overflow_action block
89
+ </buffer>
90
+
91
+ <server>
92
+ host 0.0.0.0
93
+ port 24224
94
+ </server>
95
+ </match>
96
+ ```
97
+
data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/results.html ADDED
@@ -0,0 +1,136 @@
1
+
2
+ <html>
3
+ <div>
4
+ <div><b>Options</b><div>
5
+ <div>Image: quay.io/jcantril/fluentd:marshal</div>
6
+ <div>Total Log Stressors: 1</div>
7
+ <div>Lines Per Second: 2500</div>
8
+ <div>Run Duration: 5m</div>
9
+ <div>Payload Source: synthetic</div>
10
+ </div>
11
+ <div>
12
+ Latency of logs collected based on the time the log was generated and ingested
13
+ </div>
14
+ <table border="1">
15
+ <tr>
16
+ <th>Total</th>
17
+ <th>Size</th>
18
+ <th>Elapsed</th>
19
+ <th>Mean</th>
20
+ <th>Min</th>
21
+ <th>Max</th>
22
+ <th>Median</th>
23
+ </tr>
24
+ <tr>
25
+ <th>Msg</th>
26
+ <th></th>
27
+ <th>(s)</th>
28
+ <th>(s)</th>
29
+ <th>(s)</th>
30
+ <th>(s)</th>
31
+ <th>(s)</th>
32
+ </tr>
33
+ <tr>
34
+ <td>722919</td>
35
+ <td>256</td>
36
+ <td>5m0s</td>
37
+ <td>14.322</td>
38
+ <td>1.077</td>
39
+ <td>32.992</td>
40
+ <td>13.513</td>
41
+ </tr>
42
+ </table>
43
+ <div>
44
+ <img src="cpu.png">
45
+ </div>
46
+ <div>
47
+ <img src="mem.png">
48
+ </div>
49
+ <div>
50
+ <img src="latency.png">
51
+ </div>
52
+ <div>
53
+ <img src="loss.png">
54
+ </div>
55
+ <div>
56
+ <table border="1">
57
+ <tr>
58
+ <th>Stream</th>
59
+ <th>Min Seq</th>
60
+ <th>Max Seq</th>
61
+ <th>Purged</th>
62
+ <th>Collected</th>
63
+ <th>Percent Collected</th>
64
+ </tr>
65
+ <tr>
66
+ <tr><td>loader-0</td><td>0</td><td>722918</td><td>0</td><td>722919</td><td>100.0%</td><tr>
67
+ </table>
68
+ </div>
69
+ <div>
70
+ <code style="display:block;white-space:pre-wrap">
71
+ &lt;system&gt;
72
+ log_level debug
73
+ &lt;/system&gt;
74
+
75
+ &lt;source&gt;
76
+ @type tail
77
+ @id container-input
78
+ path /var/log/pods/testhack-ksowqntt_*/loader-*/*
79
+ exclude_path [&#34;/var/log/pods/**/*/*.gz&#34;,&#34;/var/log/pods/**/*/*.tmp&#34;]
80
+ pos_file &#34;/var/lib/fluentd/pos/containers-app&#34;
81
+ refresh_interval 5
82
+ rotate_wait 5
83
+ tag kubernetes.*
84
+ read_from_head &#34;true&#34;
85
+ &lt;parse&gt;
86
+ @type regexp
87
+ expression /^(?&lt;@timestamp&gt;[^\s]+) (?&lt;stream&gt;stdout|stderr) (?&lt;logtag&gt;[F|P]) (?&lt;message&gt;.*)$/
88
+ time_format &#39;%Y-%m-%dT%H:%M:%S.%N%:z&#39;
89
+ keep_time_key true
90
+ &lt;/parse&gt;
91
+ &lt;/source&gt;
92
+
93
+ &lt;filter kubernetes.**&gt;
94
+ @type concat
95
+ key message
96
+ partial_key logtag
97
+ partial_value P
98
+ separator &#39;&#39;
99
+ &lt;/filter&gt;
100
+
101
+ &lt;filter kubernetes.**&gt;
102
+ @id kubernetes-metadata
103
+ @type kubernetes_metadata
104
+ kubernetes_url &#39;https://kubernetes.default.svc&#39;
105
+ annotation_match [&#34;^containerType\.logging\.openshift\.io\/.*$&#34;]
106
+ allow_orphans false
107
+ cache_size &#39;1000&#39;
108
+ use_journal &#39;nil&#39;
109
+ ssl_partial_chain &#39;true&#39;
110
+ &lt;/filter&gt;
111
+
112
+ &lt;match **&gt;
113
+ @type forward
114
+ heartbeat_type none
115
+ keepalive true
116
+
117
+ &lt;buffer&gt;
118
+ flush_mode interval
119
+ flush_interval 5s
120
+ flush_at_shutdown true
121
+ flush_thread_count 2
122
+ retry_type exponential_backoff
123
+ retry_wait 1s
124
+ retry_max_interval 60s
125
+ retry_forever true
126
+ overflow_action block
127
+ &lt;/buffer&gt;
128
+
129
+ &lt;server&gt;
130
+ host 0.0.0.0
131
+ port 24224
132
+ &lt;/server&gt;
133
+ &lt;/match&gt;
134
+ </code>
135
+ </div>
136
+ </html>