RubyGems - fluent-plugin-kubernetes_metadata_filter - Versions diffs - 0.1.0 → 0.2.0 - Mend

fluent-plugin-kubernetes_metadata_filter 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (9) hide show

checksums.yaml +4 -4
data/README.md +4 -3
data/fluent-plugin-kubernetes_metadata_filter.gemspec +1 -1
data/lib/fluent/plugin/filter_kubernetes_metadata.rb +23 -13
data/test/cassettes/invalid_api_server_config.yml +53 -0
data/test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml +202 -0
data/test/plugin/test.token +1 -0
data/test/plugin/test_filter_kubernetes_metadata.rb +40 -0
metadata +8 -2

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b09f5217a8e64acff136a114d222513bc0dbc6e9
-  data.tar.gz: 2d5d1e8cc206a9290f74e3428c3aa01f2c2fd46c
+  metadata.gz: caf7caa0a18360f227fb306947d74f4d099a89e9
+  data.tar.gz: 5a91b4e9e256f711eddcd774d4929b339ba76c88
 SHA512:
-  metadata.gz: a1e354e6e49a5c3461a9b038992d6f3a74cd961405672974f8127387aa00ab22d2cefa0d82722bc7245eb3a7f17a8729224b8af8581895e4556a58ac3a323f5a
-  data.tar.gz: 8b8845065dc989f610e31e72f4daaad7c0806d4163a2c257564bda38f96ce2edcaedfc1204877732da0a6e38e340d25015671fc1edcda5be8910cd9204e67d0f
+  metadata.gz: bce6f30ed8df527f77411295005c41e0fba08f4dfafd261a93f325bcb6bf53cbae95194e5cfcaa47f7ee697d25a520df164bfccbdfbb56e39453b9b8b3e39a0f
+  data.tar.gz: b91c0ccac088c23c741effcb9616d89ca36ee439a184134ef793a92a5fc296224851ff0c13ae899735d22b919da5b3bd61d1faaddbb8feb87abde097fcbc3bf5

data/README.md CHANGED Viewed

@@ -9,14 +9,15 @@
 ## Configuration
-Configuration options are:
+Configuration options for fluent.conf are:
 * `kubernetes_url` - URL to the API server. *This is required*
 * `apiVersion` - API version to use (default: `v1beta3`)
-* `client_cert` - path to a client cert file to authenticate to the API server
-* `client_key` - path to a client key file to authenticate to the API server
 * `ca_file` - path to CA file for Kubernetes server certificate validation
 * `verify_ssl` - validate SSL certificates (default: true)
+* `client_cert` - path to a client cert file to authenticate to the API server
+* `client_key` - path to a client key file to authenticate to the API server
+* `bearer_token_file` - path to a file containing the bearer token to use for authentication
 * `container_name_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the Docker container name. This must used named capture groups for `pod_container_name`, `pod_name` & `namespace` (default: `'\/?[^_]+_(?<pod_container_name>[^\.]+)[^_]+_(?<pod_name>[^_]+)_(?<namespace>[^_]+)'`)
 * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)

data/fluent-plugin-kubernetes_metadata_filter.gemspec CHANGED Viewed

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "0.1.0"
+  gem.version       = "0.2.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb CHANGED Viewed

@@ -30,10 +30,11 @@ module Fluent
     config_param :container_name_to_kubernetes_name_regexp,
                  :string,
                  :default => '\/?[^_]+_(?<pod_container_name>[^\.]+)[^_]+_(?<pod_name>[^_]+)_(?<namespace>[^_]+)'
+    config_param :bearer_token_file, :string, :default => ''
-    def self.get_metadata(pod_name, container_name, namespace)
+    def get_metadata(pod_name, container_name, namespace)
       begin
-        metadata = @@client.get_pod(pod_name, namespace)
+        metadata = @client.get_pod(pod_name, namespace)
         if metadata
           return {
             :uid => metadata['metadata']['uid'],
@@ -60,19 +61,27 @@ module Fluent
       require 'active_support/core_ext/object/blank'
       require 'lru_redux'
-      @@client = Kubeclient::Client.new @kubernetes_url, @apiVersion
+      @client = Kubeclient::Client.new @kubernetes_url, @apiVersion
-      if @client_cert.present? && @client_key.present? && @ca_file.present?
+      @client.ssl_options(
+        client_cert: @client_cert.present? ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
+        client_key: @client_key.present? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
+        ca_file: @ca_file,
+        verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+      )
-        @@client.ssl_options(
-          client_cert: OpenSSL::X509::Certificate.new(File.read(@client_cert)),
-          client_key: OpenSSL::PKey::RSA.new(File.read(@client_key)),
-          ca_file: @ca_file,
-          verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
-        )
+      if @bearer_token_file.present?
+        bearer_token = File.read(@bearer_token_file)
+        RestClient.add_before_execution_proc do |req, params|
+          req['authorization'] ||= "Bearer #{bearer_token}"
+        end
       end
-      raise Fluent::ConfigError, 'Invalid Kubernetes API endpoint' unless @@client.api_valid?
+      begin
+        @client.api_valid?
+      rescue KubeException => kube_error
+        raise Fluent::ConfigError, "Invalid Kubernetes API endpoint: #{kube_error.message}"
+      end
       @cache = LruRedux::ThreadSafeCache.new(@cache_size)
       @container_name_to_kubernetes_name_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_name_regexp)
@@ -83,10 +92,11 @@ module Fluent
       es.each {|time, record|
         if record.has_key?(:docker) && record[:docker].has_key?(:name)
+          this = self
           metadata = @cache.getset(record[:docker][:name]){
             match_data = record[:docker][:name].match(@container_name_to_kubernetes_name_regexp_compiled)
             if match_data
-              KubernetesMetadataFilter.get_metadata(
+              this.get_metadata(
                 match_data[:pod_name],
                 match_data[:pod_container_name],
                 match_data[:namespace]
@@ -104,4 +114,4 @@ module Fluent
     end
   end
-end
+end

data/test/cassettes/invalid_api_server_config.yml ADDED Viewed

@@ -0,0 +1,53 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+      Authorization:
+      - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWF
+  response:
+    status:
+      code: 401
+      message: Unauthorized
+    headers:
+      Content-Type:
+      - text/plain; charset=utf-8
+      Date:
+      - Sat, 09 May 2015 14:04:39 GMT
+      Content-Length:
+      - '13'
+    body:
+      encoding: UTF-8
+      string: |
+        Unauthorized
+    http_version:
+  recorded_at: Sat, 09 May 2015 14:04:39 GMT
+recorded_with: VCR 2.9.3

data/test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml ADDED Viewed

@@ -0,0 +1,202 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+      Authorization:
+      - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Sat, 09 May 2015 13:51:07 GMT
+      Content-Length:
+      - '67'
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "versions": [
+            "v1beta1",
+            "v1beta2",
+            "v1beta3"
+          ]
+        }
+    http_version:
+  recorded_at: Sat, 09 May 2015 13:51:07 GMT
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1beta3/namespaces/default/pods/fabric8-console-controller-98rqc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+      Authorization:
+      - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Sat, 09 May 2015 13:51:07 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Pod",
+          "apiVersion": "v1beta3",
+          "metadata": {
+            "name": "fabric8-console-controller-98rqc",
+            "generateName": "fabric8-console-controller-",
+            "namespace": "default",
+            "selfLink": "/api/v1beta3/namespaces/default/pods/fabric8-console-controller-98rqc",
+            "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
+            "resourceVersion": "3556",
+            "creationTimestamp": "2015-05-08T09:22:42Z",
+            "labels": {
+              "component": "fabric8Console"
+            }
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "openshift-cert-secrets",
+                "hostPath": null,
+                "emptyDir": null,
+                "gcePersistentDisk": null,
+                "gitRepo": null,
+                "secret": {
+                  "secretName": "openshift-cert-secrets"
+                },
+                "nfs": null,
+                "iscsi": null,
+                "glusterfs": null
+              }
+            ],
+            "containers": [
+              {
+                "name": "fabric8-console-container",
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "ports": [
+                  {
+                    "containerPort": 9090,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "OAUTH_CLIENT_ID",
+                    "value": "fabric8-console"
+                  },
+                  {
+                    "name": "OAUTH_AUTHORIZE_URI",
+                    "value": "https://localhost:8443/oauth/authorize"
+                  }
+                ],
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "openshift-cert-secrets",
+                    "readOnly": true,
+                    "mountPath": "/etc/secret-volume"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent",
+                "capabilities": {}
+              }
+            ],
+            "restartPolicy": "Always",
+            "dnsPolicy": "ClusterFirst",
+            "host": "jimmi-redhat.localnet"
+          },
+          "status": {
+            "phase": "Running",
+            "Condition": [
+              {
+                "type": "Ready",
+                "status": "True"
+              }
+            ],
+            "hostIP": "172.17.42.1",
+            "podIP": "172.17.0.4",
+            "containerStatuses": [
+              {
+                "name": "fabric8-console-container",
+                "state": {
+                  "running": {
+                    "startedAt": "2015-05-09T13:33:38Z"
+                  }
+                },
+                "lastState": {},
+                "ready": true,
+                "restartCount": 2,
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
+                "containerID": "docker://1b1d1f61c1205fe73328c75b2945e2ce05acfba2fde16299a8103fb22e9ec58a"
+              },
+              {
+                "name": "POD",
+                "state": {
+                  "termination": {
+                    "exitCode": 0,
+                    "startedAt": "2015-05-09T09:45:08Z",
+                    "finishedAt": "2015-05-09T09:47:00Z"
+                  }
+                },
+                "lastState": {},
+                "ready": false,
+                "restartCount": 1,
+                "image": "openshift/origin-pod:v0.4.4",
+                "imageID": "docker://81fa37a83ec46fed9dddf30a9c3c43f54bfe4b65dea74a4c152fd515a0bf92a8",
+                "containerID": "docker://9c8ed1a9bbb6d9804b12379654461dcd3b447438c02c52ca7f7d2504e14ae96b"
+              }
+            ]
+          }
+        }
+    http_version:
+  recorded_at: Sat, 09 May 2015 13:51:07 GMT
+recorded_with: VCR 2.9.3

data/test/plugin/test.token ADDED Viewed

	@@ -0,0 +1 @@
1	+ 12345

data/test/plugin/test_filter_kubernetes_metadata.rb CHANGED Viewed

@@ -61,6 +61,18 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         assert_equal(1, d.instance.cache_size)
       end
     end
+    test 'invalid API server config' do
+      VCR.use_cassette('invalid_api_server_config') do
+        assert_raise Fluent::ConfigError do
+          d = create_driver(%[
+            kubernetes_url https://localhost:8443
+            bearer_token_file test/plugin/test.token
+            verify_ssl false
+          ])
+        end
+      end
+    end
   end
   sub_test_case 'filter_stream' do
@@ -99,6 +111,34 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       end
     end
+    test 'with docker & kubernetes metadata using bearer token' do
+      VCR.use_cassette('kubernetes_docker_metadata_using_bearer_token') do
+        msg = {
+            :docker => {
+                :name => '/k8s_fabric8-console-container.efbd6e64_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_42cbc279'
+            }
+        }
+        es = emit(msg, %[
+          kubernetes_url https://localhost:8443
+          verify_ssl false
+          bearer_token_file test/plugin/test.token
+        ])
+        expected_kube_metadata = {
+            :kubernetes => {
+              :host => "jimmi-redhat.localnet",
+              :pod_name =>"fabric8-console-controller-98rqc",
+              :container_name => "fabric8-console-container",
+              :namespace => "default",
+              :uid => "c76927af-f563-11e4-b32d-54ee7527188d",
+              :labels => {
+                :component => "fabric8Console"
+              }
+            }
+        }
+        assert_equal(msg.merge(expected_kube_metadata), es.instance_variable_get(:@record_array)[0])
+      end
+    end
     test 'with docker metadata, non-kubernetes' do
       VCR.use_cassette('non_kubernetes_docker_metadata') do
         msg = {

metadata CHANGED Viewed

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-08 00:00:00.000000000 Z
+date: 2015-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -208,10 +208,13 @@ files:
 - circle.yml
 - fluent-plugin-kubernetes_metadata_filter.gemspec
 - lib/fluent/plugin/filter_kubernetes_metadata.rb
+- test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata.yml
+- test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
 - test/cassettes/non_kubernetes_docker_metadata.yml
 - test/cassettes/valid_kubernetes_api_server.yml
 - test/helper.rb
+- test/plugin/test.token
 - test/plugin/test_filter_kubernetes_metadata.rb
 homepage: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
 licenses:
@@ -238,8 +241,11 @@ signing_key:
 specification_version: 4
 summary: Filter plugin to add Kubernetes metadata
 test_files:
+- test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata.yml
+- test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
 - test/cassettes/non_kubernetes_docker_metadata.yml
 - test/cassettes/valid_kubernetes_api_server.yml
 - test/helper.rb
+- test/plugin/test.token
 - test/plugin/test_filter_kubernetes_metadata.rb