fluent-plugin-kubernetes_metadata_filter 0.23.0 → 0.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ad94cfeef2063842b127dcd7717b96951b35c6e4
-  data.tar.gz: 9ba28aa2447189ed598de36c64f6776c381f090c
+  metadata.gz: 718b00c62d8a5f40dd763c50b34ddc7ab4106a98
+  data.tar.gz: 4368cdeb4409e1416c7dfe12ab75b7c604c9f5a5
 SHA512:
-  metadata.gz: 2930f99ff2db361b6492504da2a032d0148213fe11e33c44fd29e78f7d73a96a867259d618522ef1002b7adfa5a9c323349e34cc25ae2e085f27667d4fab3f50
-  data.tar.gz: 150b250919002593405ee18de5024b146c4438ce4bf5a631912b85031847d6b8e97567df9c75df2a9278bfd96cd98916440ad8c7c9f9df59bd26b67f2a001895
+  metadata.gz: 6f30bf120124b195306ab89945ecc415edd773d362b830213dade8968daa2fba48df54941a13db1eba83e778057d46319af0d3e9c329776205633e8c4d4a0a5d
+  data.tar.gz: 589952266c386bf83792e58dd404826357232ac68f798cb700ad5942c6e05f04b4c3848de3e575c6ff0782ab26ce8057060c3f8b7d06ec95328e95938cf93951

data/README.md

@@ -29,6 +29,7 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
 * `use_journal` - If false (default), messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.
 * `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field (default: `'^k8s_(?<container_name>[^\.]+)\.(?<container_hash>[a-z0-9]{8})_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_(?<pod_id>[^_]+)_(?<pod_randhex>[a-z0-9]{8})$'`)
+* `annotation_match` - Array of regular expressions matching annotation field names. Matched annotations are added to a log record.
 
 Reading from the JSON formatted log files with `in_tail` and wildcard filenames:
 ```

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "0.23.0"
+  gem.version       = "0.24.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -54,6 +54,8 @@ module Fluent
                  :string,
                  :default => '^k8s_(?<container_name>[^\.]+)\.[^_]+_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_[^_]+_[a-f0-9]{8}$'
 
+    config_param :annotation_match, :array, default: []
+
     def syms_to_strs(hsh)
       newhsh = {}
       hsh.each_pair do |kk,vv|
@@ -74,10 +76,11 @@ module Fluent
         metadata = @client.get_pod(pod_name, namespace_name)
         return if !metadata
         labels = syms_to_strs(metadata['metadata']['labels'].to_h)
+        annotations = match_annotations(syms_to_strs(metadata['metadata']['annotations'].to_h))
         if @de_dot
           self.de_dot!(labels)
         end
-        return {
+        kubernetes_metadata = {
             'namespace_name' => namespace_name,
             'pod_id'         => metadata['metadata']['uid'],
             'pod_name'       => pod_name,
@@ -85,6 +88,8 @@ module Fluent
             'labels'         => labels,
             'host'           => metadata['spec']['nodeName']
         }
+        kubernetes_metadata['annotations'] = annotations unless annotations.empty?
+        return kubernetes_metadata
       rescue KubeException
         nil
       end
@@ -180,6 +185,16 @@ module Fluent
         @merge_json_log_key = 'log'
         self.class.class_eval { alias_method :filter_stream, :filter_stream_from_files }
       end
+
+      @annotations_regexps = []
+      @annotation_match.each do |regexp|
+        begin
+          @annotations_regexps << Regexp.compile(regexp)
+        rescue RegexpError => e
+          log.error "Error: invalid regular expression in annotation_match: #{e}"
+        end
+      end
+
     end
 
     def filter_stream_from_files(tag, es)
@@ -324,6 +339,18 @@ module Fluent
       end
     end
 
+    def match_annotations(annotations)
+      result = {}
+      @annotations_regexps.each do |regexp|
+        annotations.each do |key, value|
+          if ::Fluent::StringUtil.match_regexp(regexp, key.to_s)
+            result[key] = value
+          end
+        end
+      end
+      result
+    end
+
     def start_watch
       begin
         resource_version = @client.get_pods.resourceVersion
@@ -341,13 +368,15 @@ module Fluent
                 cache_key = "#{pod_cache_key}_#{container_status['name']}"
                 cached    = @cache[cache_key]
                 if cached
-                  # Only thing that can be modified is labels
+                  # Only thing that can be modified is labels and (possibly) annotations
                   labels = syms_to_strs(notice.object.metadata.labels.to_h)
+                  annotations = match_annotations(syms_to_strs(notice.object.metadata.annotations.to_h))
                   if @de_dot
                     self.de_dot!(labels)
                   end
                   cached['kubernetes']['labels'] = labels
-                  @cache[cache_key]            = cached
+                  cached['kubernetes']['annotations'] = annotations unless annotations.empty?
+                  @cache[cache_key] = cached
                 end
               }
             end

data/test/cassettes/kubernetes_docker_metadata_annotations.yml

@@ -0,0 +1,188 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Content-Length:
+      - '67'
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "versions": [
+            "v1"
+          ]
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Pod",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "fabric8-console-controller-98rqc",
+            "generateName": "fabric8-console-controller-",
+            "namespace": "default",
+            "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
+            "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
+            "resourceVersion": "122",
+            "creationTimestamp": "2015-05-08T09:22:42Z",
+            "labels": {
+              "component": "fabric8Console"
+            },
+            "annotations": {
+              "kubernetes.io/config.hash": "c171c44f5b9345c6dc17b0e95030318c",
+              "kubernetes.io/config.mirror": "c171c44f5b9345c6dc17b0e95030318c",
+              "kubernetes.io/config.seen": "2016-06-06T08:08:35.680437994Z",
+              "kubernetes.io/config.source": "file",
+              "custom.field1": "hello_kitty",
+              "field.two": "value"
+            }
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "openshift-cert-secrets",
+                "hostPath": null,
+                "emptyDir": null,
+                "gcePersistentDisk": null,
+                "gitRepo": null,
+                "secret": {
+                  "secretName": "openshift-cert-secrets"
+                },
+                "nfs": null,
+                "iscsi": null,
+                "glusterfs": null
+              }
+            ],
+            "containers": [
+              {
+                "name": "fabric8-console-container",
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "ports": [
+                  {
+                    "containerPort": 9090,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "OAUTH_CLIENT_ID",
+                    "value": "fabric8-console"
+                  },
+                  {
+                    "name": "OAUTH_AUTHORIZE_URI",
+                    "value": "https://localhost:8443/oauth/authorize"
+                  }
+                ],
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "openshift-cert-secrets",
+                    "readOnly": true,
+                    "mountPath": "/etc/secret-volume"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent",
+                "capabilities": {}
+              }
+            ],
+            "restartPolicy": "Always",
+            "dnsPolicy": "ClusterFirst",
+            "nodeName": "jimmi-redhat.localnet"
+          },
+          "status": {
+            "phase": "Running",
+            "Condition": [
+              {
+                "type": "Ready",
+                "status": "True"
+              }
+            ],
+            "hostIP": "172.17.42.1",
+            "podIP": "172.17.0.8",
+            "containerStatuses": [
+              {
+                "name": "fabric8-console-container",
+                "state": {
+                  "running": {
+                    "startedAt": "2015-05-08T09:22:44Z"
+                  }
+                },
+                "lastState": {},
+                "ready": true,
+                "restartCount": 0,
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
+                "containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459"
+              }
+            ]
+          }
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -502,5 +502,36 @@ use_journal true
       end
     end
 
+    test 'with kubernetes annotations' do
+      VCR.use_cassette('kubernetes_docker_metadata_annotations') do
+        es = emit({},'
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+          annotation_match [ "^custom.+", "two"]
+        ')
+        expected_kube_metadata = {
+            'docker' => {
+                'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+            },
+            'kubernetes' => {
+                'host'           => 'jimmi-redhat.localnet',
+                'pod_name'       => 'fabric8-console-controller-98rqc',
+                'container_name' => 'fabric8-console-container',
+                'namespace_name' => 'default',
+                'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
+                'labels'         => {
+                    'component' => 'fabric8Console'
+                },
+                'annotations'    => {
+                    'custom.field1' => 'hello_kitty',
+                    'field.two' => 'value'
+                }
+            }
+        }
+        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+      end
+    end
+
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 0.23.0
+  version: 0.24.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-14 00:00:00.000000000 Z
+date: 2016-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -223,6 +223,7 @@ files:
 - lib/fluent/plugin/filter_kubernetes_metadata.rb
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata.yml
+- test/cassettes/kubernetes_docker_metadata_annotations.yml
 - test/cassettes/kubernetes_docker_metadata_dotted_labels.yml
 - test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
 - test/cassettes/metadata_with_namespace_id.yml
@@ -251,13 +252,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Filter plugin to add Kubernetes metadata
 test_files:
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata.yml
+- test/cassettes/kubernetes_docker_metadata_annotations.yml
 - test/cassettes/kubernetes_docker_metadata_dotted_labels.yml
 - test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
 - test/cassettes/metadata_with_namespace_id.yml