RubyGems - fluent-plugin-kubernetes_metadata_filter - Versions diffs - 2.1.4 → 2.9.4 - Mend

fluent-plugin-kubernetes_metadata_filter 2.1.4 → 2.9.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (40) hide show

checksums.yaml +4 -4
data/.circleci/config.yml +57 -0
data/.gitignore +0 -1
data/.rubocop.yml +57 -0
data/Gemfile +4 -2
data/Gemfile.lock +158 -0
data/README.md +48 -28
data/Rakefile +15 -11
data/fluent-plugin-kubernetes_metadata_filter.gemspec +25 -28
data/lib/fluent/plugin/filter_kubernetes_metadata.rb +185 -131
data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +27 -20
data/lib/fluent/plugin/kubernetes_metadata_common.rb +59 -33
data/lib/fluent/plugin/kubernetes_metadata_stats.rb +6 -6
data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
data/lib/fluent/plugin/kubernetes_metadata_util.rb +53 -0
data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +121 -27
data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +138 -29
data/release_notes.md +42 -0
data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -34
data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -34
data/test/cassettes/kubernetes_get_api_v1.yml +193 -0
data/test/cassettes/kubernetes_get_api_v1_using_token.yml +195 -0
data/test/cassettes/kubernetes_get_namespace_default.yml +69 -0
data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +71 -0
data/test/cassettes/{kubernetes_docker_metadata.yml → kubernetes_get_pod.yml} +0 -82
data/test/cassettes/{metadata_with_namespace_id.yml → kubernetes_get_pod_container_init.yml} +3 -134
data/test/cassettes/{kubernetes_docker_metadata_using_bearer_token.yml → kubernetes_get_pod_using_token.yml} +5 -105
data/test/cassettes/metadata_from_tag_and_journald_fields.yml +0 -255
data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +0 -255
data/test/cassettes/{non_kubernetes_docker_metadata.yml → valid_kubernetes_api_server_using_token.yml} +4 -44
data/test/helper.rb +20 -2
data/test/plugin/test_cache_stats.rb +10 -13
data/test/plugin/test_cache_strategy.rb +158 -160
data/test/plugin/test_filter_kubernetes_metadata.rb +480 -320
data/test/plugin/test_utils.rb +56 -0
data/test/plugin/test_watch_namespaces.rb +209 -55
data/test/plugin/test_watch_pods.rb +302 -103
data/test/plugin/watch_test.rb +52 -33
metadata +69 -72
data/circle.yml +0 -17

data/release_notes.md ADDED Viewed

@@ -0,0 +1,42 @@
+# Release Notes
+## 2.9.4
+As of this release, the 'de_dot' functionality is depricated and will be removed in future releases.
+Ref: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/issues/320
+## v2.1.4
+The use of `use_journal` is **DEPRECATED**.  If this setting is not present, the plugin will
+attempt to figure out the source of the metadata fields from the following:
+- If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
+`docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
+then the plugin will use those values as the source to use to lookup the metadata
+- If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
+then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
+- Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
+lookup the metdata
+## v2.1.x
+As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
+payload.  The following configuration options are removed:
+* `merge_json_log`
+* `preserve_json_log`
+One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser).
+It can parsed with the parser plugin like this:
+```
+<filter kubernetes.**>
+  @type parser
+  key_name log
+  <parse>
+    @type json
+    json_parser json
+  </parse>
+  replace_invalid_sequence true
+  reserve_data true # this preserves unparsable log lines
+  emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean
+  reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time.
+</filter>
+```

data/test/cassettes/kubernetes_docker_metadata_annotations.yml CHANGED Viewed

@@ -18,40 +18,6 @@
 #
 ---
 http_interactions:
-- request:
-    method: get
-    uri: https://localhost:8443/api
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-      User-Agent:
-      - Ruby
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Type:
-      - application/json
-      Date:
-      - Fri, 08 May 2015 10:35:37 GMT
-      Content-Length:
-      - '67'
-    body:
-      encoding: UTF-8
-      string: |-
-        {
-          "versions": [
-            "v1"
-          ]
-        }
-    http_version:
-  recorded_at: Fri, 08 May 2015 10:35:37 GMT
 - request:
     method: get
     uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc

data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} RENAMED Viewed

@@ -18,40 +18,6 @@
 #
 ---
 http_interactions:
-- request:
-    method: get
-    uri: https://localhost:8443/api
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-      User-Agent:
-      - Ruby
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Type:
-      - application/json
-      Date:
-      - Fri, 08 May 2015 10:35:37 GMT
-      Content-Length:
-      - '67'
-    body:
-      encoding: UTF-8
-      string: |-
-        {
-          "versions": [
-            "v1"
-          ]
-        }
-    http_version:
-  recorded_at: Fri, 08 May 2015 10:35:37 GMT
 - request:
     method: get
     uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc

data/test/cassettes/kubernetes_get_api_v1.yml ADDED Viewed

@@ -0,0 +1,193 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "APIResourceList",
+          "groupVersion": "v1",
+          "resources": [
+            {
+              "name": "bindings",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Binding",
+              "verbs": [
+                "create"
+              ]
+            },
+            {
+              "name": "namespaces",
+              "singularName": "",
+              "namespaced": false,
+              "kind": "Namespace",
+              "verbs": [
+                "create",
+                "delete",
+                "get",
+                "list",
+                "patch",
+                "update",
+                "watch"
+              ],
+              "shortNames": [
+                "ns"
+              ]
+            },
+            {
+              "name": "namespaces/finalize",
+              "singularName": "",
+              "namespaced": false,
+              "kind": "Namespace",
+              "verbs": [
+                "update"
+              ]
+            },
+            {
+              "name": "namespaces/status",
+              "singularName": "",
+              "namespaced": false,
+              "kind": "Namespace",
+              "verbs": [
+                "get",
+                "patch",
+                "update"
+              ]
+            },
+            {
+              "name": "pods",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": [
+                "create",
+                "delete",
+                "deletecollection",
+                "get",
+                "list",
+                "patch",
+                "update",
+                "watch"
+              ],
+              "shortNames": [
+                "po"
+              ],
+              "categories": [
+                "all"
+              ]
+            },
+            {
+              "name": "pods/attach",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": []
+            },
+            {
+              "name": "pods/binding",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Binding",
+              "verbs": [
+                "create"
+              ]
+            },
+            {
+              "name": "pods/eviction",
+              "singularName": "",
+              "namespaced": true,
+              "group": "policy",
+              "version": "v1beta1",
+              "kind": "Eviction",
+              "verbs": [
+                "create"
+              ]
+            },
+            {
+              "name": "pods/exec",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": []
+            },
+            {
+              "name": "pods/log",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": [
+                "get"
+              ]
+            },
+            {
+              "name": "pods/portforward",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": []
+            },
+            {
+              "name": "pods/proxy",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": []
+            },
+            {
+              "name": "pods/status",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": [
+                "get",
+                "patch",
+                "update"
+              ]
+            }
+          ]
+        }
+    http_version:
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3

data/test/cassettes/kubernetes_get_api_v1_using_token.yml ADDED Viewed

@@ -0,0 +1,195 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+      Authorization:
+      - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "APIResourceList",
+          "groupVersion": "v1",
+          "resources": [
+            {
+              "name": "bindings",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Binding",
+              "verbs": [
+                "create"
+              ]
+            },
+            {
+              "name": "namespaces",
+              "singularName": "",
+              "namespaced": false,
+              "kind": "Namespace",
+              "verbs": [
+                "create",
+                "delete",
+                "get",
+                "list",
+                "patch",
+                "update",
+                "watch"
+              ],
+              "shortNames": [
+                "ns"
+              ]
+            },
+            {
+              "name": "namespaces/finalize",
+              "singularName": "",
+              "namespaced": false,
+              "kind": "Namespace",
+              "verbs": [
+                "update"
+              ]
+            },
+            {
+              "name": "namespaces/status",
+              "singularName": "",
+              "namespaced": false,
+              "kind": "Namespace",
+              "verbs": [
+                "get",
+                "patch",
+                "update"
+              ]
+            },
+            {
+              "name": "pods",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": [
+                "create",
+                "delete",
+                "deletecollection",
+                "get",
+                "list",
+                "patch",
+                "update",
+                "watch"
+              ],
+              "shortNames": [
+                "po"
+              ],
+              "categories": [
+                "all"
+              ]
+            },
+            {
+              "name": "pods/attach",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": []
+            },
+            {
+              "name": "pods/binding",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Binding",
+              "verbs": [
+                "create"
+              ]
+            },
+            {
+              "name": "pods/eviction",
+              "singularName": "",
+              "namespaced": true,
+              "group": "policy",
+              "version": "v1beta1",
+              "kind": "Eviction",
+              "verbs": [
+                "create"
+              ]
+            },
+            {
+              "name": "pods/exec",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": []
+            },
+            {
+              "name": "pods/log",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": [
+                "get"
+              ]
+            },
+            {
+              "name": "pods/portforward",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": []
+            },
+            {
+              "name": "pods/proxy",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": []
+            },
+            {
+              "name": "pods/status",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": [
+                "get",
+                "patch",
+                "update"
+              ]
+            }
+          ]
+        }
+    http_version:
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3

data/test/cassettes/kubernetes_get_namespace_default.yml ADDED Viewed

@@ -0,0 +1,69 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Namespace",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "default",
+            "selfLink": "/api/v1/namespaces/default",
+            "uid": "898268c8-4a36-11e5-9d81-42010af0194c",
+            "resourceVersion": "6",
+            "creationTimestamp": "2015-05-08T09:22:01Z"
+          },
+          "spec": {
+            "finalizers": [
+                "kubernetes"
+            ]
+          },
+          "status": {
+            "phase": "Active"
+          }
+        }
+    http_version:
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3

data/test/cassettes/kubernetes_get_namespace_default_using_token.yml ADDED Viewed

@@ -0,0 +1,71 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+      Authorization:
+      - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Namespace",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "default",
+            "selfLink": "/api/v1/namespaces/default",
+            "uid": "898268c8-4a36-11e5-9d81-42010af0194c",
+            "resourceVersion": "6",
+            "creationTimestamp": "2015-05-08T09:22:01Z"
+          },
+          "spec": {
+            "finalizers": [
+                "kubernetes"
+            ]
+          },
+          "status": {
+            "phase": "Active"
+          }
+        }
+    http_version:
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3