fluent-plugin-kubernetes_metadata_filter 2.8.0 → 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +2 -6
  3. data/.gitignore +0 -1
  4. data/Gemfile.lock +45 -45
  5. data/README.md +5 -83
  6. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/cpu.png +0 -0
  7. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/latency.png +0 -0
  8. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/loss.png +0 -0
  9. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/mem.png +0 -0
  10. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/readme.md +88 -0
  11. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/results.html +127 -0
  12. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/cpu.png +0 -0
  13. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/latency.png +0 -0
  14. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/loss.png +0 -0
  15. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/mem.png +0 -0
  16. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/readme.md +97 -0
  17. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/results.html +136 -0
  18. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/cpu.png +0 -0
  19. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/latency.png +0 -0
  20. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/loss.png +0 -0
  21. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/mem.png +0 -0
  22. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/readme.md +97 -0
  23. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/results.html +136 -0
  24. data/fluent-plugin-kubernetes_metadata_filter.gemspec +6 -6
  25. data/lib/fluent/plugin/filter_kubernetes_metadata.rb +102 -127
  26. data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +3 -5
  27. data/lib/fluent/plugin/kubernetes_metadata_common.rb +9 -15
  28. data/lib/fluent/plugin/kubernetes_metadata_stats.rb +16 -0
  29. data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +2 -2
  30. data/lib/fluent/plugin/kubernetes_metadata_util.rb +0 -20
  31. data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +33 -0
  32. data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +33 -0
  33. data/release_notes.md +42 -0
  34. data/test/cassettes/kubernetes_get_pod_container_init.yml +145 -0
  35. data/test/plugin/test_cache_strategy.rb +1 -1
  36. data/test/plugin/test_filter_kubernetes_metadata.rb +108 -421
  37. data/test/plugin/watch_test.rb +2 -0
  38. metadata +36 -9
  39. data/test/plugin/test_utils.rb +0 -56
  40. /data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 452a507fb0cbb14f8eb17fe488d07a8ccd9b9f97bef38b592e56a5bb8acbe3bb
4
- data.tar.gz: ff1346d50b7b31ebfd7105523eec621baea0db0a771c193d48094b3c8818e502
3
+ metadata.gz: 239811c2b14fb09de2809bd9151b8c4b6cfb2cfa1cc4a62fdc12c2fedf707b66
4
+ data.tar.gz: 35b9f7b4d1f6dfe992c1187d81ac5024503162101f8f742e3378e82f56918d0e
5
5
  SHA512:
6
- metadata.gz: 3f9257176000157861bf21980b91e0594494118d4d5cb10eee475127567c4803865e3c1404c02c1e035c015c264d0a3351660ed031566c5d399f774088205f21
7
- data.tar.gz: 439308c8a534c7e0c5cdcf70f8ccd26eeb4ab455abacff1d411f86ae57e4da7b6b422c42f772d84508360a3fec60ed415c454633fecde854b4c288d24cd6d39b
6
+ metadata.gz: e9f98d31986a3c3af094b28da85a8fa9671897dccdf738ffe6245c3cc0a24f4e8b1592637f37d7b0622f1fda0577fee597ccd2aec3634442bec05535c817a0d5
7
+ data.tar.gz: e56741d6c9af9dd77b28bd3f20be72d6b8bb2c04318551b53081b81fde720fe0c10acbef79624fc6bafc39a5904eb845b839aaba1704f054b325697c70615f07
data/.circleci/config.yml CHANGED
@@ -4,7 +4,8 @@ install: &install
4
4
  name: Install bundle
5
5
  command: |
6
6
  gem install bundler
7
- bundle install --path vendor/bundle
7
+ bundle config set --local path vendor/bundle
8
+ bundle install
8
9
 
9
10
  missingdeps: &missingdeps
10
11
  name: Install missing dependecies
@@ -20,9 +21,6 @@ test: &test
20
21
  command: bundle exec rake test --trace
21
22
 
22
23
  executors:
23
- ruby-2-5:
24
- docker:
25
- - image: circleci/ruby:2.5.5
26
24
  ruby-2-6:
27
25
  docker:
28
26
  - image: circleci/ruby:2.6.3
@@ -49,8 +47,6 @@ jobs:
49
47
  workflows:
50
48
  "test_multiple_ruby_versions":
51
49
  jobs:
52
- - ruby-test:
53
- ruby-version: ruby-2-5
54
50
  - ruby-test:
55
51
  ruby-version: ruby-2-6
56
52
  - ruby-test:
data/.gitignore CHANGED
@@ -7,7 +7,6 @@ vendor/
7
7
  InstalledFiles
8
8
  _yardoc
9
9
  coverage
10
- doc/
11
10
  lib/bundler/man
12
11
  pkg
13
12
  rdoc
data/Gemfile.lock CHANGED
@@ -1,9 +1,9 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fluent-plugin-kubernetes_metadata_filter (2.8.0)
5
- fluentd (>= 0.14.0, < 1.15)
6
- kubeclient (< 5)
4
+ fluent-plugin-kubernetes_metadata_filter (3.0.0)
5
+ fluentd (>= 0.14.0, < 1.16)
6
+ kubeclient (>= 4.0.0, < 5.0.0)
7
7
  lru_redux
8
8
 
9
9
  GEM
@@ -11,42 +11,42 @@ GEM
11
11
  specs:
12
12
  addressable (2.8.0)
13
13
  public_suffix (>= 2.0.2, < 5.0)
14
- ast (2.4.1)
14
+ ast (2.4.2)
15
15
  bump (0.10.0)
16
16
  charlock_holmes (0.7.7)
17
17
  codeclimate-test-reporter (0.6.0)
18
18
  simplecov (>= 0.7.1, < 1.0.0)
19
- concurrent-ruby (1.1.9)
19
+ concurrent-ruby (1.1.10)
20
20
  cool.io (1.7.1)
21
21
  copyright-header (1.0.22)
22
22
  github-linguist
23
23
  crack (0.4.5)
24
24
  rexml
25
- docile (1.3.5)
25
+ docile (1.4.0)
26
26
  domain_name (0.5.20190701)
27
27
  unf (>= 0.0.5, < 1.0.0)
28
- escape_utils (1.2.1)
29
- ffi (1.15.3)
28
+ escape_utils (1.2.2)
29
+ ffi (1.15.5)
30
30
  ffi-compiler (1.0.1)
31
31
  ffi (>= 1.0.0)
32
32
  rake
33
- fluentd (1.14.0)
33
+ fluentd (1.15.0)
34
34
  bundler
35
35
  cool.io (>= 1.4.5, < 2.0.0)
36
- http_parser.rb (>= 0.5.1, < 0.8.0)
36
+ http_parser.rb (>= 0.5.1, < 0.9.0)
37
37
  msgpack (>= 1.3.1, < 2.0.0)
38
- serverengine (>= 2.2.2, < 3.0.0)
38
+ serverengine (>= 2.3.0, < 3.0.0)
39
39
  sigdump (~> 0.2.2)
40
- strptime (>= 0.2.2, < 1.0.0)
40
+ strptime (>= 0.2.4, < 1.0.0)
41
41
  tzinfo (>= 1.0, < 3.0)
42
42
  tzinfo-data (~> 1.0)
43
43
  webrick (>= 1.4.2, < 1.8.0)
44
44
  yajl-ruby (~> 1.0)
45
- github-linguist (7.12.2)
45
+ github-linguist (7.21.0)
46
46
  charlock_holmes (~> 0.7.7)
47
47
  escape_utils (~> 1.2.0)
48
48
  mini_mime (~> 1.0)
49
- rugged (>= 0.25.1)
49
+ rugged (~> 1.0)
50
50
  hashdiff (1.0.1)
51
51
  http (4.4.1)
52
52
  addressable (~> 2.3)
@@ -54,58 +54,58 @@ GEM
54
54
  http-form_data (~> 2.2)
55
55
  http-parser (~> 1.2.0)
56
56
  http-accept (1.7.0)
57
- http-cookie (1.0.4)
57
+ http-cookie (1.0.5)
58
58
  domain_name (~> 0.5)
59
59
  http-form_data (2.3.0)
60
60
  http-parser (1.2.3)
61
61
  ffi-compiler (>= 1.0, < 2.0)
62
- http_parser.rb (0.7.0)
63
- jsonpath (1.1.0)
62
+ http_parser.rb (0.8.0)
63
+ jsonpath (1.1.2)
64
64
  multi_json
65
- kubeclient (4.9.2)
65
+ kubeclient (4.9.3)
66
66
  http (>= 3.0, < 5.0)
67
67
  jsonpath (~> 1.0)
68
68
  recursive-open-struct (~> 1.1, >= 1.1.1)
69
69
  rest-client (~> 2.0)
70
70
  lru_redux (1.1.0)
71
- mime-types (3.3.1)
71
+ mime-types (3.4.1)
72
72
  mime-types-data (~> 3.2015)
73
- mime-types-data (3.2021.0704)
74
- mini_mime (1.0.2)
73
+ mime-types-data (3.2022.0105)
74
+ mini_mime (1.1.2)
75
75
  minitest (4.7.5)
76
- msgpack (1.4.2)
76
+ msgpack (1.5.3)
77
77
  multi_json (1.15.0)
78
78
  netrc (0.11.0)
79
- parallel (1.20.1)
80
- parser (3.0.0.0)
79
+ parallel (1.22.1)
80
+ parser (3.1.2.0)
81
81
  ast (~> 2.4.1)
82
- power_assert (1.2.0)
83
- public_suffix (4.0.6)
84
- rainbow (3.0.0)
85
- rake (13.0.3)
82
+ power_assert (2.0.1)
83
+ public_suffix (4.0.7)
84
+ rainbow (3.1.1)
85
+ rake (13.0.6)
86
86
  recursive-open-struct (1.1.3)
87
- regexp_parser (2.0.3)
87
+ regexp_parser (2.5.0)
88
88
  rest-client (2.1.0)
89
89
  http-accept (>= 1.7.0, < 2.0)
90
90
  http-cookie (>= 1.0.2, < 2.0)
91
91
  mime-types (>= 1.16, < 4.0)
92
92
  netrc (~> 0.8)
93
93
  rexml (3.2.5)
94
- rr (1.2.1)
95
- rubocop (1.8.1)
94
+ rr (3.0.9)
95
+ rubocop (1.28.2)
96
96
  parallel (~> 1.10)
97
- parser (>= 3.0.0.0)
97
+ parser (>= 3.1.0.0)
98
98
  rainbow (>= 2.2.2, < 4.0)
99
99
  regexp_parser (>= 1.8, < 3.0)
100
100
  rexml
101
- rubocop-ast (>= 1.2.0, < 2.0)
101
+ rubocop-ast (>= 1.17.0, < 2.0)
102
102
  ruby-progressbar (~> 1.7)
103
103
  unicode-display_width (>= 1.4.0, < 3.0)
104
- rubocop-ast (1.4.0)
105
- parser (>= 2.7.1.5)
104
+ rubocop-ast (1.17.0)
105
+ parser (>= 3.1.1.0)
106
106
  ruby-progressbar (1.11.0)
107
- rugged (1.1.0)
108
- serverengine (2.2.4)
107
+ rugged (1.4.3)
108
+ serverengine (2.3.0)
109
109
  sigdump (~> 0.2.2)
110
110
  sigdump (0.2.4)
111
111
  simplecov (0.21.2)
@@ -113,7 +113,7 @@ GEM
113
113
  simplecov-html (~> 0.11)
114
114
  simplecov_json_formatter (~> 0.1)
115
115
  simplecov-html (0.12.3)
116
- simplecov_json_formatter (0.1.2)
116
+ simplecov_json_formatter (0.1.4)
117
117
  strptime (0.2.5)
118
118
  test-unit (3.0.9)
119
119
  power_assert
@@ -122,19 +122,19 @@ GEM
122
122
  test-unit (>= 2.5.2)
123
123
  tzinfo (2.0.4)
124
124
  concurrent-ruby (~> 1.0)
125
- tzinfo-data (1.2021.1)
125
+ tzinfo-data (1.2022.1)
126
126
  tzinfo (>= 1.0.0)
127
127
  unf (0.1.4)
128
128
  unf_ext
129
- unf_ext (0.0.7.7)
130
- unicode-display_width (2.0.0)
129
+ unf_ext (0.0.8.2)
130
+ unicode-display_width (2.2.0)
131
131
  vcr (6.0.0)
132
- webmock (3.11.1)
133
- addressable (>= 2.3.6)
132
+ webmock (3.14.0)
133
+ addressable (>= 2.8.0)
134
134
  crack (>= 0.3.2)
135
135
  hashdiff (>= 0.4.0, < 2.0.0)
136
136
  webrick (1.7.0)
137
- yajl-ruby (1.4.1)
137
+ yajl-ruby (1.4.3)
138
138
 
139
139
  PLATFORMS
140
140
  ruby
@@ -155,4 +155,4 @@ DEPENDENCIES
155
155
  yajl-ruby
156
156
 
157
157
  BUNDLED WITH
158
- 2.2.19
158
+ 2.3.16
data/README.md CHANGED
@@ -7,8 +7,7 @@
7
7
 
8
8
  The Kubernetes metadata plugin filter enriches container log records with pod and namespace metadata.
9
9
 
10
- This plugin derives basic metadata about the container that emitted a given log record using the source of the log record. Records from journald provide metadata about the
11
- container environment as named fields. Records from JSON files encode metadata about the container in the file name. The initial metadata derived from the source is used
10
+ This plugin derives basic metadata about the container that emitted a given log record using the source of the log record. Records from kubernetes containers encode metadata about the container in the file name. The initial metadata derived from the source is used
12
11
  to lookup additional metadata about the container's associated pod and namespace (e.g. UUIDs, labels, annotations) when the kubernetes_url is configured. If the plugin cannot
13
12
  authoritatively determine the namespace of the container emitting a log record, it will use an 'orphan' namespace ID in the metadata. This behaviors supports multi-tenant systems
14
13
  that rely on the authenticity of the namespace for proper log isolation.
@@ -17,6 +16,7 @@ that rely on the authenticity of the namespace for proper log isolation.
17
16
 
18
17
  | fluent-plugin-kubernetes_metadata_filter | fluentd | ruby |
19
18
  |-------------------|---------|------|
19
+ | >= 2.10.0 | >= v1.10.0 | >= 2.6 |
20
20
  | >= 2.5.0 | >= v1.10.0 | >= 2.5 |
21
21
  | >= 2.0.0 | >= v0.14.20 | >= 2.1 |
22
22
  | < 2.0.0 | >= v0.12.0 | >= 1.9 |
@@ -41,16 +41,10 @@ Configuration options for fluent.conf are:
41
41
  * `client_key` - path to a client key file to authenticate to the API server
42
42
  * `bearer_token_file` - path to a file containing the bearer token to use for authentication
43
43
  * `tag_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the current fluentd tag.
44
- This must used named capture groups for `container_name`, `pod_name` & `namespace` default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L52)
44
+ This must use named capture groups for `container_name`, `pod_name`, `namespace`, and either `pod_uuid (/var/log/pods)` or `docker_id (/var/log/containers)`
45
45
  * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
46
46
  * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
47
47
  * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
48
- * `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
49
- * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
50
- * *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename. If true, messages are expected to be formatted as if read from the systemd journal. The `MESSAGE` field has the full message. The `CONTAINER_NAME` field has the encoded k8s metadata (see below). The `CONTAINER_ID_FULL` field has the full container uuid. This requires docker to use the `--log-driver=journald` log driver. If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
51
- if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
52
- * `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)
53
- * This corresponds to the definition [in the source](https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317)
54
48
  * `annotation_match` - Array of regular expressions matching annotation field names. Matched annotations are added to a log record.
55
49
  * `allow_orphans` - Modify the namespace and namespace id to the values of `orphaned_namespace_name` and `orphaned_namespace_id`
56
50
  when true (default: `true`)
@@ -64,27 +58,11 @@ when true (default: `true`)
64
58
  * `skip_container_metadata` - Skip some of the container data of the metadata. The metadata will not contain the container_image and container_image_id fields.
65
59
  * `skip_master_url` - Skip the master_url field from the metadata.
66
60
  * `skip_namespace_metadata` - Skip the namespace_id field from the metadata. The fetch_namespace_metadata function will be skipped. The plugin will be faster and cpu consumption will be less.
61
+ * `stats_interval` - The interval to display cache stats (default: 30s). Set to 0 to disable stats collection and logging
67
62
  * `watch_retry_interval` - The time interval in seconds for retry backoffs when watch connections fail. (default: `10`)
68
63
 
69
- **NOTE:** As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
70
- payload. The following configuration options are removed:
71
64
 
72
- * `merge_json_log`
73
- * `preserve_json_log`
74
-
75
- One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser)
76
-
77
- **NOTE** As of this release, the use of `use_journal` is **DEPRECATED**. If this setting is not present, the plugin will
78
- attempt to figure out the source of the metadata fields from the following:
79
- - If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
80
- `docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
81
- then the plugin will use those values as the source to use to lookup the metadata
82
- - If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
83
- then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
84
- - Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
85
- lookup the metdata
86
-
87
- Reading from the JSON formatted log files with `in_tail` and wildcard filenames while respecting the CRI-o log format with the same config you need the fluent-plugin "multi-format-parser":
65
+ Reading from a JSON formatted log files with `in_tail` and wildcard filenames while respecting the CRI-o log format with the same config you need the fluent-plugin "multi-format-parser":
88
66
 
89
67
  ```
90
68
  fluent-gem install fluent-plugin-multi-format-parser
@@ -125,50 +103,6 @@ The config block could look like this:
125
103
  </match>
126
104
  ```
127
105
 
128
- Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` and `systemd-journal` plugins, and requires docker to use the `--log-driver=journald` log driver):
129
- ```
130
- <source>
131
- @type systemd
132
- path /run/log/journal
133
- pos_file journal.pos
134
- tag journal
135
- read_from_head true
136
- </source>
137
-
138
- # probably want to use something like fluent-plugin-rewrite-tag-filter to
139
- # retag entries from k8s
140
- <match journal>
141
- @type rewrite_tag_filter
142
- rewriterule1 CONTAINER_NAME ^k8s_ kubernetes.journal.container
143
- ...
144
- </match>
145
-
146
- <filter kubernetes.**>
147
- @type kubernetes_metadata
148
- use_journal true
149
- </filter>
150
-
151
- <match **>
152
- @type stdout
153
- </match>
154
- ```
155
- ## Log content as JSON
156
- In former versions this plugin parsed the value of the key log as JSON. In the current version this feature was removed, to avoid duplicate features in the fluentd plugin ecosystem. It can parsed with the parser plugin like this:
157
- ```
158
- <filter kubernetes.**>
159
- @type parser
160
- key_name log
161
- <parse>
162
- @type json
163
- json_parser json
164
- </parse>
165
- replace_invalid_sequence true
166
- reserve_data true # this preserves unparsable log lines
167
- emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean
168
- reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time.
169
- </filter>
170
- ```
171
-
172
106
  ## Environment variables for Kubernetes
173
107
 
174
108
  If the name of the Kubernetes node the plugin is running on is set as
@@ -228,18 +162,6 @@ Then output becomes as belows
228
162
  }
229
163
  ```
230
164
 
231
- If using journal input, from docker configured with `--log-driver=journald`, the input looks like the `journalctl -o export` format:
232
- ```
233
- # The stream identification is encoded into the PRIORITY field as an
234
- # integer: 6, or github.com/coreos/go-systemd/journal.Info, marks stdout,
235
- # while 3, or github.com/coreos/go-systemd/journal.Err, marks stderr.
236
- PRIORITY=6
237
- CONTAINER_ID=b6cbb6e73c0a
238
- CONTAINER_ID_FULL=b6cbb6e73c0ad63ab820e4baa97cdc77cec729930e38a714826764ac0491341a
239
- CONTAINER_NAME=k8s_registry.a49f5318_docker-registry-1-hhoj0_default_ae3a9bdc-1f66-11e6-80a2-fa163e2fff3a_799e4035
240
- MESSAGE=172.17.0.1 - - [21/May/2016:16:52:05 +0000] "GET /healthz HTTP/1.1" 200 0 "" "Go-http-client/1.1"
241
- ```
242
-
243
165
  ## Contributing
244
166
 
245
167
  1. Fork it
@@ -0,0 +1,88 @@
1
+
2
+ # collector Functionl Benchmark Results
3
+ ## Options
4
+ * Image: quay.io/openshift-logging/fluentd:1.14.6
5
+ * Total Log Stressors: 1
6
+ * Lines Per Second: 2500
7
+ * Run Duration: 5m
8
+ * Payload Source: synthetic
9
+
10
+ ## Latency of logs collected based on the time the log was generated and ingested
11
+
12
+ Total Msg| Size | Elapsed (s) | Mean (s)| Min(s) | Max (s)| Median (s)
13
+ ---------|------|-------------|---------|--------|--------|---
14
+ 705733|256|5m0s|16.929|1.826|36.399|14.956
15
+
16
+ ![](cpu.png)
17
+
18
+ ![](mem.png)
19
+
20
+ ![](latency.png)
21
+
22
+ ![](loss.png)
23
+
24
+ ## Percent logs lost between first and last collected sequence ids
25
+ Stream | Min Seq | Max Seq | Purged | Collected | Percent Collected |
26
+ -------| ---------| --------| -------|-----------|--------------|
27
+ | functional.0.0000000000000000DBCED7977B8F643F|0|716850|0|705733|98.4%
28
+
29
+
30
+ ## Config
31
+
32
+ ```
33
+
34
+ <system>
35
+ log_level debug
36
+ </system>
37
+
38
+ <source>
39
+ @type tail
40
+ @id container-input
41
+ path /var/log/pods/testhack-clwn63sc_*/loader-*/*
42
+ exclude_path ["/var/log/pods/**/*/*.gz","/var/log/pods/**/*/*.tmp"]
43
+ pos_file "/var/lib/fluentd/pos/containers-app"
44
+ refresh_interval 5
45
+ rotate_wait 5
46
+ tag kubernetes.*
47
+ read_from_head "true"
48
+ <parse>
49
+ @type regexp
50
+ expression /^(?<@timestamp>[^\s]+) (?<stream>stdout|stderr) (?<logtag>[F|P]) (?<message>.*)$/
51
+ time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
52
+ keep_time_key true
53
+ </parse>
54
+ </source>
55
+
56
+ <filter kubernetes.**>
57
+ @type concat
58
+ key message
59
+ partial_key logtag
60
+ partial_value P
61
+ separator ''
62
+ </filter>
63
+
64
+ <match **>
65
+ @type forward
66
+ heartbeat_type none
67
+ keepalive true
68
+
69
+ <buffer>
70
+ flush_mode interval
71
+ flush_interval 5s
72
+ flush_at_shutdown true
73
+ flush_thread_count 2
74
+ retry_type exponential_backoff
75
+ retry_wait 1s
76
+ retry_max_interval 60s
77
+ retry_forever true
78
+ overflow_action block
79
+ </buffer>
80
+
81
+ <server>
82
+ host 0.0.0.0
83
+ port 24224
84
+ </server>
85
+ </match>
86
+
87
+ ```
88
+
@@ -0,0 +1,127 @@
1
+
2
+ <html>
3
+ <div>
4
+ <div><b>Options</b><div>
5
+ <div>Image: quay.io/openshift-logging/fluentd:1.14.6</div>
6
+ <div>Total Log Stressors: 1</div>
7
+ <div>Lines Per Second: 2500</div>
8
+ <div>Run Duration: 5m</div>
9
+ <div>Payload Source: synthetic</div>
10
+ </div>
11
+ <div>
12
+ Latency of logs collected based on the time the log was generated and ingested
13
+ </div>
14
+ <table border="1">
15
+ <tr>
16
+ <th>Total</th>
17
+ <th>Size</th>
18
+ <th>Elapsed</th>
19
+ <th>Mean</th>
20
+ <th>Min</th>
21
+ <th>Max</th>
22
+ <th>Median</th>
23
+ </tr>
24
+ <tr>
25
+ <th>Msg</th>
26
+ <th></th>
27
+ <th>(s)</th>
28
+ <th>(s)</th>
29
+ <th>(s)</th>
30
+ <th>(s)</th>
31
+ <th>(s)</th>
32
+ </tr>
33
+ <tr>
34
+ <td>705733</td>
35
+ <td>256</td>
36
+ <td>5m0s</td>
37
+ <td>16.929</td>
38
+ <td>1.826</td>
39
+ <td>36.399</td>
40
+ <td>14.956</td>
41
+ </tr>
42
+ </table>
43
+ <div>
44
+ <img src="cpu.png">
45
+ </div>
46
+ <div>
47
+ <img src="mem.png">
48
+ </div>
49
+ <div>
50
+ <img src="latency.png">
51
+ </div>
52
+ <div>
53
+ <img src="loss.png">
54
+ </div>
55
+ <div>
56
+ <table border="1">
57
+ <tr>
58
+ <th>Stream</th>
59
+ <th>Min Seq</th>
60
+ <th>Max Seq</th>
61
+ <th>Purged</th>
62
+ <th>Collected</th>
63
+ <th>Percent Collected</th>
64
+ </tr>
65
+ <tr>
66
+ <tr><td>functional.0.0000000000000000DBCED7977B8F643F</td><td>0</td><td>716850</td><td>0</td><td>705733</td><td>98.4%</td><tr>
67
+ </table>
68
+ </div>
69
+ <div>
70
+ <code style="display:block;white-space:pre-wrap">
71
+
72
+ &lt;system&gt;
73
+ log_level debug
74
+ &lt;/system&gt;
75
+
76
+ &lt;source&gt;
77
+ @type tail
78
+ @id container-input
79
+ path /var/log/pods/testhack-clwn63sc_*/loader-*/*
80
+ exclude_path [&#34;/var/log/pods/**/*/*.gz&#34;,&#34;/var/log/pods/**/*/*.tmp&#34;]
81
+ pos_file &#34;/var/lib/fluentd/pos/containers-app&#34;
82
+ refresh_interval 5
83
+ rotate_wait 5
84
+ tag kubernetes.*
85
+ read_from_head &#34;true&#34;
86
+ &lt;parse&gt;
87
+ @type regexp
88
+ expression /^(?&lt;@timestamp&gt;[^\s]+) (?&lt;stream&gt;stdout|stderr) (?&lt;logtag&gt;[F|P]) (?&lt;message&gt;.*)$/
89
+ time_format &#39;%Y-%m-%dT%H:%M:%S.%N%:z&#39;
90
+ keep_time_key true
91
+ &lt;/parse&gt;
92
+ &lt;/source&gt;
93
+
94
+ &lt;filter kubernetes.**&gt;
95
+ @type concat
96
+ key message
97
+ partial_key logtag
98
+ partial_value P
99
+ separator &#39;&#39;
100
+ &lt;/filter&gt;
101
+
102
+ &lt;match **&gt;
103
+ @type forward
104
+ heartbeat_type none
105
+ keepalive true
106
+
107
+ &lt;buffer&gt;
108
+ flush_mode interval
109
+ flush_interval 5s
110
+ flush_at_shutdown true
111
+ flush_thread_count 2
112
+ retry_type exponential_backoff
113
+ retry_wait 1s
114
+ retry_max_interval 60s
115
+ retry_forever true
116
+ overflow_action block
117
+ &lt;/buffer&gt;
118
+
119
+ &lt;server&gt;
120
+ host 0.0.0.0
121
+ port 24224
122
+ &lt;/server&gt;
123
+ &lt;/match&gt;
124
+
125
+ </code>
126
+ </div>
127
+ </html>