fluent-plugin-kubernetes_metadata_filter 2.1.4 → 2.9.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +57 -0
  3. data/.gitignore +0 -1
  4. data/.rubocop.yml +57 -0
  5. data/Gemfile +4 -2
  6. data/Gemfile.lock +158 -0
  7. data/README.md +48 -28
  8. data/Rakefile +15 -11
  9. data/fluent-plugin-kubernetes_metadata_filter.gemspec +25 -28
  10. data/lib/fluent/plugin/filter_kubernetes_metadata.rb +185 -131
  11. data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +27 -20
  12. data/lib/fluent/plugin/kubernetes_metadata_common.rb +59 -33
  13. data/lib/fluent/plugin/kubernetes_metadata_stats.rb +6 -6
  14. data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
  15. data/lib/fluent/plugin/kubernetes_metadata_util.rb +53 -0
  16. data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +121 -27
  17. data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +138 -29
  18. data/release_notes.md +42 -0
  19. data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -34
  20. data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -34
  21. data/test/cassettes/kubernetes_get_api_v1.yml +193 -0
  22. data/test/cassettes/kubernetes_get_api_v1_using_token.yml +195 -0
  23. data/test/cassettes/kubernetes_get_namespace_default.yml +69 -0
  24. data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +71 -0
  25. data/test/cassettes/{kubernetes_docker_metadata.yml → kubernetes_get_pod.yml} +0 -82
  26. data/test/cassettes/{metadata_with_namespace_id.yml → kubernetes_get_pod_container_init.yml} +3 -134
  27. data/test/cassettes/{kubernetes_docker_metadata_using_bearer_token.yml → kubernetes_get_pod_using_token.yml} +5 -105
  28. data/test/cassettes/metadata_from_tag_and_journald_fields.yml +0 -255
  29. data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +0 -255
  30. data/test/cassettes/{non_kubernetes_docker_metadata.yml → valid_kubernetes_api_server_using_token.yml} +4 -44
  31. data/test/helper.rb +20 -2
  32. data/test/plugin/test_cache_stats.rb +10 -13
  33. data/test/plugin/test_cache_strategy.rb +158 -160
  34. data/test/plugin/test_filter_kubernetes_metadata.rb +480 -320
  35. data/test/plugin/test_utils.rb +56 -0
  36. data/test/plugin/test_watch_namespaces.rb +209 -55
  37. data/test/plugin/test_watch_pods.rb +302 -103
  38. data/test/plugin/watch_test.rb +52 -33
  39. metadata +69 -72
  40. data/circle.yml +0 -17
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 0762066546f02f63127244e84c7c44e7f70b83737bd6abc27ab37f4a01be1fa8
4
- data.tar.gz: 11372dfe77e89ed7c2bc2b0b4a2d48a51ea2a132b8b6e1cd9b1538ed4f8f3b3b
3
+ metadata.gz: 98c4002df3754262c547e0b22acf366e62bca72f0142a20430f6a0f6c91dadea
4
+ data.tar.gz: 3f9e5d2d19b505c7cad15d113b69dea7ccb0943a3d4605d121fdf2234952c4a7
5
5
  SHA512:
6
- metadata.gz: ea2b022e7da9df5da158ce02f24c187fc605e6c4da8b7aecea9d0847cfac9d29fa91c59a9748b32643f3e40baffc9974cefe153e74e171b33d02a74dbb856d44
7
- data.tar.gz: dfd3d105bcb61a94edf76c3862fd6bbd615f5634e1c3fd3a621aac246f6f17186fcaddf703a59a4201b286d39a19b8f9b3eed532213820b0d70419f0b06f9021
6
+ metadata.gz: 5474457d9a35d7951a8a380c0099deeb16af6d849a9f036c467ba47e0fcc47ce2ff7f77bbd75ad7d6487c9f172897e63406a91fd9f08dc6494eb32332e2b9373
7
+ data.tar.gz: 91a14c8edabcdd1eaa7b915c14bfd643b7829ae0cc8951ed8f9e77819483638207a5c6264094974971dcf4d7684c50c62a249cee4de36d23574b72cbe4e10589
@@ -0,0 +1,57 @@
1
+ version: 2.1
2
+
3
+ install: &install
4
+ name: Install bundle
5
+ command: |
6
+ gem install bundler
7
+ bundle install --path vendor/bundle
8
+
9
+ missingdeps: &missingdeps
10
+ name: Install missing dependecies
11
+ command: |
12
+ cat /etc/os-release
13
+ printf "deb http://deb.debian.org/debian buster main\ndeb http://security.debian.org buster/updates main\ndeb-src http://security.debian.org buster/updates main" > /tmp/sources.list
14
+ sudo cp /tmp/sources.list /etc/apt/sources.list
15
+ sudo apt-get --allow-releaseinfo-change update
16
+ sudo apt-get install cmake libicu-dev libssl-dev
17
+
18
+ test: &test
19
+ name: Test bundle
20
+ command: bundle exec rake test --trace
21
+
22
+ executors:
23
+ ruby-2-5:
24
+ docker:
25
+ - image: circleci/ruby:2.5.5
26
+ ruby-2-6:
27
+ docker:
28
+ - image: circleci/ruby:2.6.3
29
+ ruby-2-7:
30
+ docker:
31
+ - image: circleci/ruby:2.7.1
32
+ jobs:
33
+ "ruby-test":
34
+ parameters:
35
+ ruby-version:
36
+ type: executor
37
+ executor: << parameters.ruby-version >>
38
+ working_directory: ~/fluent-plugin-kubernetes_metadata_filter
39
+ steps:
40
+ - run: *missingdeps
41
+ - checkout
42
+ - run: *install
43
+ - run: *test
44
+ - store_test_results:
45
+ path: coverage
46
+ - store_artifacts:
47
+ path: coverage
48
+
49
+ workflows:
50
+ "test_multiple_ruby_versions":
51
+ jobs:
52
+ - ruby-test:
53
+ ruby-version: ruby-2-5
54
+ - ruby-test:
55
+ ruby-version: ruby-2-6
56
+ - ruby-test:
57
+ ruby-version: ruby-2-7
data/.gitignore CHANGED
@@ -4,7 +4,6 @@
4
4
  .config
5
5
  .yardoc
6
6
  vendor/
7
- Gemfile.lock
8
7
  InstalledFiles
9
8
  _yardoc
10
9
  coverage
data/.rubocop.yml ADDED
@@ -0,0 +1,57 @@
1
+ AllCops:
2
+ TargetRubyVersion: 2.5 # keep in sync with .circleci/config.yml and gemspec
3
+ NewCops: enable
4
+
5
+ Style/EmptyMethod:
6
+ Enabled: false
7
+
8
+ Metrics:
9
+ Enabled: false
10
+
11
+ # not safe ... needs require 'English'
12
+ Style/SpecialGlobalVars:
13
+ Enabled: false
14
+
15
+ Layout/LineLength:
16
+ Max: 205 # TODO: lower
17
+
18
+ Style/Documentation:
19
+ Enabled: false
20
+
21
+ Naming/AccessorMethodName:
22
+ Enabled: false
23
+
24
+ Naming/MethodParameterName:
25
+ Enabled: false
26
+
27
+ Style/IfInsideElse:
28
+ Enabled: false
29
+
30
+ Style/GuardClause:
31
+ Enabled: false
32
+
33
+ Lint/NestedMethodDefinition:
34
+ Enabled: false
35
+
36
+ # TODO: fix
37
+ Style/StringConcatenation:
38
+ Enabled: false
39
+
40
+ Style/NumericPredicate:
41
+ EnforcedStyle: comparison
42
+
43
+ Style/IfUnlessModifier:
44
+ Enabled: false
45
+
46
+ Style/ClassAndModuleChildren:
47
+ Enabled: false
48
+
49
+ # TODO: enable ... somehow breaks tests
50
+ Style/HashEachMethods:
51
+ Enabled: false
52
+
53
+ Style/WordArray:
54
+ EnforcedStyle: brackets
55
+
56
+ Style/SymbolArray:
57
+ EnforcedStyle: brackets
data/Gemfile CHANGED
@@ -1,7 +1,9 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source 'https://rubygems.org'
2
4
 
3
- gem 'codeclimate-test-reporter', '<1.0.0', :group => :test, :require => nil
4
- gem 'rubocop', require: false
5
+ gem 'codeclimate-test-reporter', '<1.0.0', group: :test, require: nil
6
+ gem 'rubocop'
5
7
 
6
8
  # Specify your gem's dependencies in fluent-plugin-add.gemspec
7
9
  gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,158 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ fluent-plugin-kubernetes_metadata_filter (2.9.4)
5
+ fluentd (>= 0.14.0, < 1.15)
6
+ kubeclient (>= 4.0.0, < 5.0.0)
7
+ lru_redux
8
+
9
+ GEM
10
+ remote: https://rubygems.org/
11
+ specs:
12
+ addressable (2.8.0)
13
+ public_suffix (>= 2.0.2, < 5.0)
14
+ ast (2.4.2)
15
+ bump (0.10.0)
16
+ charlock_holmes (0.7.7)
17
+ codeclimate-test-reporter (0.6.0)
18
+ simplecov (>= 0.7.1, < 1.0.0)
19
+ concurrent-ruby (1.1.9)
20
+ cool.io (1.7.1)
21
+ copyright-header (1.0.22)
22
+ github-linguist
23
+ crack (0.4.5)
24
+ rexml
25
+ docile (1.4.0)
26
+ domain_name (0.5.20190701)
27
+ unf (>= 0.0.5, < 1.0.0)
28
+ escape_utils (1.2.1)
29
+ ffi (1.15.4)
30
+ ffi-compiler (1.0.1)
31
+ ffi (>= 1.0.0)
32
+ rake
33
+ fluentd (1.14.3)
34
+ bundler
35
+ cool.io (>= 1.4.5, < 2.0.0)
36
+ http_parser.rb (>= 0.5.1, < 0.9.0)
37
+ msgpack (>= 1.3.1, < 2.0.0)
38
+ serverengine (>= 2.2.2, < 3.0.0)
39
+ sigdump (~> 0.2.2)
40
+ strptime (>= 0.2.4, < 1.0.0)
41
+ tzinfo (>= 1.0, < 3.0)
42
+ tzinfo-data (~> 1.0)
43
+ webrick (>= 1.4.2, < 1.8.0)
44
+ yajl-ruby (~> 1.0)
45
+ github-linguist (7.17.0)
46
+ charlock_holmes (~> 0.7.7)
47
+ escape_utils (~> 1.2.0)
48
+ mini_mime (~> 1.0)
49
+ rugged (>= 0.25.1)
50
+ hashdiff (1.0.1)
51
+ http (4.4.1)
52
+ addressable (~> 2.3)
53
+ http-cookie (~> 1.0)
54
+ http-form_data (~> 2.2)
55
+ http-parser (~> 1.2.0)
56
+ http-accept (1.7.0)
57
+ http-cookie (1.0.4)
58
+ domain_name (~> 0.5)
59
+ http-form_data (2.3.0)
60
+ http-parser (1.2.3)
61
+ ffi-compiler (>= 1.0, < 2.0)
62
+ http_parser.rb (0.8.0)
63
+ jsonpath (1.1.0)
64
+ multi_json
65
+ kubeclient (4.9.2)
66
+ http (>= 3.0, < 5.0)
67
+ jsonpath (~> 1.0)
68
+ recursive-open-struct (~> 1.1, >= 1.1.1)
69
+ rest-client (~> 2.0)
70
+ lru_redux (1.1.0)
71
+ mime-types (3.4.1)
72
+ mime-types-data (~> 3.2015)
73
+ mime-types-data (3.2021.1115)
74
+ mini_mime (1.1.2)
75
+ minitest (4.7.5)
76
+ msgpack (1.4.2)
77
+ multi_json (1.15.0)
78
+ netrc (0.11.0)
79
+ parallel (1.21.0)
80
+ parser (3.0.2.0)
81
+ ast (~> 2.4.1)
82
+ power_assert (2.0.1)
83
+ public_suffix (4.0.6)
84
+ rainbow (3.0.0)
85
+ rake (13.0.6)
86
+ recursive-open-struct (1.1.3)
87
+ regexp_parser (2.1.1)
88
+ rest-client (2.1.0)
89
+ http-accept (>= 1.7.0, < 2.0)
90
+ http-cookie (>= 1.0.2, < 2.0)
91
+ mime-types (>= 1.16, < 4.0)
92
+ netrc (~> 0.8)
93
+ rexml (3.2.5)
94
+ rr (3.0.8)
95
+ rubocop (1.22.3)
96
+ parallel (~> 1.10)
97
+ parser (>= 3.0.0.0)
98
+ rainbow (>= 2.2.2, < 4.0)
99
+ regexp_parser (>= 1.8, < 3.0)
100
+ rexml
101
+ rubocop-ast (>= 1.12.0, < 2.0)
102
+ ruby-progressbar (~> 1.7)
103
+ unicode-display_width (>= 1.4.0, < 3.0)
104
+ rubocop-ast (1.12.0)
105
+ parser (>= 3.0.1.1)
106
+ ruby-progressbar (1.11.0)
107
+ rugged (1.2.0)
108
+ serverengine (2.2.4)
109
+ sigdump (~> 0.2.2)
110
+ sigdump (0.2.4)
111
+ simplecov (0.21.2)
112
+ docile (~> 1.1)
113
+ simplecov-html (~> 0.11)
114
+ simplecov_json_formatter (~> 0.1)
115
+ simplecov-html (0.12.3)
116
+ simplecov_json_formatter (0.1.3)
117
+ strptime (0.2.5)
118
+ test-unit (3.0.9)
119
+ power_assert
120
+ test-unit-rr (1.0.5)
121
+ rr (>= 1.1.1)
122
+ test-unit (>= 2.5.2)
123
+ tzinfo (2.0.4)
124
+ concurrent-ruby (~> 1.0)
125
+ tzinfo-data (1.2021.5)
126
+ tzinfo (>= 1.0.0)
127
+ unf (0.1.4)
128
+ unf_ext
129
+ unf_ext (0.0.8)
130
+ unicode-display_width (2.1.0)
131
+ vcr (6.0.0)
132
+ webmock (3.14.0)
133
+ addressable (>= 2.8.0)
134
+ crack (>= 0.3.2)
135
+ hashdiff (>= 0.4.0, < 2.0.0)
136
+ webrick (1.7.0)
137
+ yajl-ruby (1.4.1)
138
+
139
+ PLATFORMS
140
+ ruby
141
+
142
+ DEPENDENCIES
143
+ bump
144
+ bundler (~> 2.0)
145
+ codeclimate-test-reporter (< 1.0.0)
146
+ copyright-header
147
+ fluent-plugin-kubernetes_metadata_filter!
148
+ minitest (~> 4.0)
149
+ rake
150
+ rubocop
151
+ test-unit (~> 3.0.2)
152
+ test-unit-rr (~> 1.0.3)
153
+ vcr
154
+ webmock
155
+ yajl-ruby
156
+
157
+ BUNDLED WITH
158
+ 2.3.4
data/README.md CHANGED
@@ -2,6 +2,8 @@
2
2
  [![Circle CI](https://circleci.com/gh/fabric8io/fluent-plugin-kubernetes_metadata_filter.svg?style=svg)](https://circleci.com/gh/fabric8io/fluent-plugin-kubernetes_metadata_filter)
3
3
  [![Code Climate](https://codeclimate.com/github/fabric8io/fluent-plugin-kubernetes_metadata_filter/badges/gpa.svg)](https://codeclimate.com/github/fabric8io/fluent-plugin-kubernetes_metadata_filter)
4
4
  [![Test Coverage](https://codeclimate.com/github/fabric8io/fluent-plugin-kubernetes_metadata_filter/badges/coverage.svg)](https://codeclimate.com/github/fabric8io/fluent-plugin-kubernetes_metadata_filter)
5
+ [![Ruby Style Guide](https://img.shields.io/badge/code_style-rubocop-brightgreen.svg)](https://github.com/rubocop-hq/rubocop)
6
+ [![Ruby Style Guide](https://img.shields.io/badge/code_style-community-brightgreen.svg)](https://rubystyle.guide)
5
7
 
6
8
  The Kubernetes metadata plugin filter enriches container log records with pod and namespace metadata.
7
9
 
@@ -15,6 +17,7 @@ that rely on the authenticity of the namespace for proper log isolation.
15
17
 
16
18
  | fluent-plugin-kubernetes_metadata_filter | fluentd | ruby |
17
19
  |-------------------|---------|------|
20
+ | >= 2.5.0 | >= v1.10.0 | >= 2.5 |
18
21
  | >= 2.0.0 | >= v0.14.20 | >= 2.1 |
19
22
  | < 2.0.0 | >= v0.12.0 | >= 1.9 |
20
23
 
@@ -38,64 +41,79 @@ Configuration options for fluent.conf are:
38
41
  * `client_key` - path to a client key file to authenticate to the API server
39
42
  * `bearer_token_file` - path to a file containing the bearer token to use for authentication
40
43
  * `tag_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the current fluentd tag.
41
- This must used named capture groups for `container_name`, `pod_name` & `namespace` (default: `\.(?<pod_name>[^\._]+)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$</pod>)`)
44
+ This must use named capture groups for `container_name`, `pod_name`, `namespace`, and either `pod_uuid (/var/log/pods)` or `docker_id (/var/log/containers)`
42
45
  * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
43
46
  * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
44
47
  * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
45
- * `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
46
- * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
48
+ * *DEPRECATED*`de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for Datadog and ElasticSearch 2.x compatibility (default: `true`)
49
+ * *DEPRECATED*`de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
50
+ * *DEPRECATED*`de_slash` - replace slashes in labels and annotations with configured `de_slash_separator`, required for Datadog compatibility (default: `false`)
51
+ * *DEPRECATED*`de_slash_separator` - separator to use if `de_slash` is enabled (default: `__`)
47
52
  * *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename. If true, messages are expected to be formatted as if read from the systemd journal. The `MESSAGE` field has the full message. The `CONTAINER_NAME` field has the encoded k8s metadata (see below). The `CONTAINER_ID_FULL` field has the full container uuid. This requires docker to use the `--log-driver=journald` log driver. If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
48
53
  if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
49
- * `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field (default: `'^(?<name_prefix>[^_]+)_(?<container_name>[^\._]+)(\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_[^_]+_[^_]+$'`
50
- * This corresponds to the definition [in the source](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/dockertools/docker.go#L317)
54
+ * `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)
55
+ * This corresponds to the definition [in the source](https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317)
51
56
  * `annotation_match` - Array of regular expressions matching annotation field names. Matched annotations are added to a log record.
52
57
  * `allow_orphans` - Modify the namespace and namespace id to the values of `orphaned_namespace_name` and `orphaned_namespace_id`
53
58
  when true (default: `true`)
54
59
  * `orphaned_namespace_name` - The namespace to associate with records where the namespace can not be determined (default: `.orphaned`)
55
60
  * `orphaned_namespace_id` - The namespace id to associate with records where the namespace can not be determined (default: `orphaned`)
56
61
  * `lookup_from_k8s_field` - If the field `kubernetes` is present, lookup the metadata from the given subfields such as `kubernetes.namespace_name`, `kubernetes.pod_name`, etc. This allows you to avoid having to pass in metadata to lookup in an explicitly formatted tag name or in an explicitly formatted `CONTAINER_NAME` value. For example, set `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`, and `docker.id` in the record, and the filter will fill in the rest. (default: `true`)
62
+ * `ssl_partial_chain` - if `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want
63
+ to trust the intermediate CA certs we do have, set this to `true` - this corresponds to
64
+ the `openssl s_client -partial_chain` flag and `X509_V_FLAG_PARTIAL_CHAIN` (default: `false`)
65
+ * `skip_labels` - Skip all label fields from the metadata.
66
+ * `skip_container_metadata` - Skip some of the container data of the metadata. The metadata will not contain the container_image and container_image_id fields.
67
+ * `skip_master_url` - Skip the master_url field from the metadata.
68
+ * `skip_namespace_metadata` - Skip the namespace_id field from the metadata. The fetch_namespace_metadata function will be skipped. The plugin will be faster and cpu consumption will be less.
69
+ * `watch_retry_interval` - The time interval in seconds for retry backoffs when watch connections fail. (default: `10`)
57
70
 
58
- **NOTE:** As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
59
- payload. The following configuration options are removed:
60
71
 
61
- * `merge_json_log`
62
- * `preserve_json_log`
72
+ Reading from the JSON formatted log files with `in_tail` and wildcard filenames while respecting the CRI-o log format with the same config you need the fluent-plugin "multi-format-parser":
63
73
 
64
- **NOTE** As of this release, the use of `use_journal` is **DEPRECATED**. If this setting is not present, the plugin will
65
- attempt to figure out the source of the metadata fields from the following:
66
- - If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
67
- `docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
68
- then the plugin will use those values as the source to use to lookup the metadata
69
- - If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
70
- then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
71
- - Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
72
- lookup the metdata
74
+ ```
75
+ fluent-gem install fluent-plugin-multi-format-parser
76
+ ```
73
77
 
74
- Reading from the JSON formatted log files with `in_tail` and wildcard filenames:
78
+ The config block could look like this:
75
79
  ```
76
80
  <source>
77
- type tail
81
+ @type tail
78
82
  path /var/log/containers/*.log
79
83
  pos_file fluentd-docker.pos
80
- time_format %Y-%m-%dT%H:%M:%S
81
- tag kubernetes.*
82
- format json
83
84
  read_from_head true
85
+ tag kubernetes.*
86
+ <parse>
87
+ @type multi_format
88
+ <pattern>
89
+ format json
90
+ time_key time
91
+ time_type string
92
+ time_format "%Y-%m-%dT%H:%M:%S.%NZ"
93
+ keep_time_key false
94
+ </pattern>
95
+ <pattern>
96
+ format regexp
97
+ expression /^(?<time>.+) (?<stream>stdout|stderr)( (?<logtag>.))? (?<log>.*)$/
98
+ time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
99
+ keep_time_key false
100
+ </pattern>
101
+ </parse>
84
102
  </source>
85
103
 
86
104
  <filter kubernetes.var.log.containers.**.log>
87
- type kubernetes_metadata
105
+ @type kubernetes_metadata
88
106
  </filter>
89
107
 
90
108
  <match **>
91
- type stdout
109
+ @type stdout
92
110
  </match>
93
111
  ```
94
112
 
95
113
  Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` and `systemd-journal` plugins, and requires docker to use the `--log-driver=journald` log driver):
96
114
  ```
97
115
  <source>
98
- type systemd
116
+ @type systemd
99
117
  path /run/log/journal
100
118
  pos_file journal.pos
101
119
  tag journal
@@ -111,15 +129,16 @@ Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` a
111
129
  </match>
112
130
 
113
131
  <filter kubernetes.**>
114
- type kubernetes_metadata
132
+ @type kubernetes_metadata
115
133
  use_journal true
116
134
  </filter>
117
135
 
118
136
  <match **>
119
- type stdout
137
+ @type stdout
120
138
  </match>
121
139
  ```
122
140
 
141
+
123
142
  ## Environment variables for Kubernetes
124
143
 
125
144
  If the name of the Kubernetes node the plugin is running on is set as
@@ -162,6 +181,7 @@ Then output becomes as belows
162
181
  "host": "jimmi-redhat.localnet",
163
182
  "pod_name":"fabric8-console-controller-98rqc",
164
183
  "pod_id": "c76927af-f563-11e4-b32d-54ee7527188d",
184
+ "pod_ip": "172.17.0.8",
165
185
  "container_name": "fabric8-console-container",
166
186
  "namespace_name": "default",
167
187
  "namespace_id": "23437884-8e08-4d95-850b-e94378c9b2fd",
data/Rakefile CHANGED
@@ -1,10 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/setup'
1
4
  require 'bundler/gem_tasks'
2
5
  require 'rake/testtask'
3
6
  require 'bump/tasks'
7
+ require 'rubocop/rake_task'
4
8
 
5
- task :test => [:base_test]
9
+ task test: [:base_test]
10
+ task default: [:test, :build, :rubocop]
6
11
 
7
- task :default => [:test, :build]
12
+ RuboCop::RakeTask.new
8
13
 
9
14
  desc 'Run test_unit based test'
10
15
  Rake::TestTask.new(:base_test) do |t|
@@ -13,7 +18,6 @@ Rake::TestTask.new(:base_test) do |t|
13
18
  # $ bundle exec rake base_test TEST=test/test_*.rb
14
19
  t.libs << 'test'
15
20
  t.test_files = Dir['test/**/test_*.rb'].sort
16
- #t.verbose = true
17
21
  t.warning = false
18
22
  end
19
23
 
@@ -23,15 +27,15 @@ task :headers do
23
27
  require 'copyright_header'
24
28
 
25
29
  args = {
26
- :license => 'Apache-2.0',
27
- :copyright_software => 'Fluentd Kubernetes Metadata Filter Plugin',
28
- :copyright_software_description => 'Enrich Fluentd events with Kubernetes metadata',
29
- :copyright_holders => ['Red Hat, Inc.'],
30
- :copyright_years => ['2015-2017'],
31
- :add_path => 'lib:test',
32
- :output_dir => '.'
30
+ license: 'Apache-2.0',
31
+ copyright_software: 'Fluentd Kubernetes Metadata Filter Plugin',
32
+ copyright_software_description: 'Enrich Fluentd events with Kubernetes metadata',
33
+ copyright_holders: ['Red Hat, Inc.'],
34
+ copyright_years: ['2015-2021'],
35
+ add_path: 'lib:test',
36
+ output_dir: '.'
33
37
  }
34
38
 
35
- command_line = CopyrightHeader::CommandLine.new( args )
39
+ command_line = CopyrightHeader::CommandLine.new(args)
36
40
  command_line.execute
37
41
  end
@@ -1,37 +1,34 @@
1
- # coding: utf-8
2
- lib = File.expand_path('../lib', __FILE__)
1
+ # frozen_string_literal: true
2
+
3
+ lib = File.expand_path('lib', __dir__)
3
4
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
5
 
5
6
  Gem::Specification.new do |gem|
6
- gem.name = "fluent-plugin-kubernetes_metadata_filter"
7
- gem.version = "2.1.4"
8
- gem.authors = ["Jimmi Dyson"]
9
- gem.email = ["jimmidyson@gmail.com"]
10
- gem.description = %q{Filter plugin to add Kubernetes metadata}
11
- gem.summary = %q{Fluentd filter plugin to add Kubernetes metadata}
12
- gem.homepage = "https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter"
13
- gem.license = "Apache-2.0"
7
+ gem.name = 'fluent-plugin-kubernetes_metadata_filter'
8
+ gem.version = '2.9.4'
9
+ gem.authors = ['Jimmi Dyson']
10
+ gem.email = ['jimmidyson@gmail.com']
11
+ gem.description = 'Filter plugin to add Kubernetes metadata'
12
+ gem.summary = 'Fluentd filter plugin to add Kubernetes metadata'
13
+ gem.homepage = 'https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter'
14
+ gem.license = 'Apache-2.0'
14
15
 
15
16
  gem.files = `git ls-files`.split($/)
16
- gem.executables = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
- gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
18
- gem.require_paths = ["lib"]
19
- gem.has_rdoc = false
20
17
 
21
- gem.required_ruby_version = '>= 2.1.0'
18
+ gem.required_ruby_version = '>= 2.5.0'
22
19
 
23
- gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
24
- gem.add_runtime_dependency "lru_redux"
25
- gem.add_runtime_dependency "kubeclient", "~> 1.1.4"
20
+ gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.15']
21
+ gem.add_runtime_dependency 'kubeclient', ['>= 4.0.0', '< 5.0.0']
22
+ gem.add_runtime_dependency 'lru_redux'
26
23
 
27
- gem.add_development_dependency "bundler", "~> 1.3"
28
- gem.add_development_dependency "rake"
29
- gem.add_development_dependency "minitest", "~> 4.0"
30
- gem.add_development_dependency "test-unit", "~> 3.0.2"
31
- gem.add_development_dependency "test-unit-rr", "~> 1.0.3"
32
- gem.add_development_dependency "copyright-header"
33
- gem.add_development_dependency "webmock"
34
- gem.add_development_dependency "vcr"
35
- gem.add_development_dependency "bump"
36
- gem.add_development_dependency "yajl-ruby"
24
+ gem.add_development_dependency 'bump'
25
+ gem.add_development_dependency 'bundler', '~> 2.0'
26
+ gem.add_development_dependency 'copyright-header'
27
+ gem.add_development_dependency 'minitest', '~> 4.0'
28
+ gem.add_development_dependency 'rake'
29
+ gem.add_development_dependency 'test-unit', '~> 3.0.2'
30
+ gem.add_development_dependency 'test-unit-rr', '~> 1.0.3'
31
+ gem.add_development_dependency 'vcr'
32
+ gem.add_development_dependency 'webmock'
33
+ gem.add_development_dependency 'yajl-ruby'
37
34
  end