RubyGems - fluent-plugin-kubernetes_metadata_filter - Versions diffs - 0.1.0 → 0.2.0 - Mend

fluent-plugin-kubernetes_metadata_filter 0.1.0 → 0.2.0

Files changed (9) hide show

checksums.yaml +4 -4
data/README.md +4 -3
data/fluent-plugin-kubernetes_metadata_filter.gemspec +1 -1
data/lib/fluent/plugin/filter_kubernetes_metadata.rb +23 -13
data/test/cassettes/invalid_api_server_config.yml +53 -0
data/test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml +202 -0
data/test/plugin/test.token +1 -0
data/test/plugin/test_filter_kubernetes_metadata.rb +40 -0
metadata +8 -2

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b09f5217a8e64acff136a114d222513bc0dbc6e9
-  data.tar.gz: 2d5d1e8cc206a9290f74e3428c3aa01f2c2fd46c
+  metadata.gz: caf7caa0a18360f227fb306947d74f4d099a89e9
+  data.tar.gz: 5a91b4e9e256f711eddcd774d4929b339ba76c88
 SHA512:
-  metadata.gz: a1e354e6e49a5c3461a9b038992d6f3a74cd961405672974f8127387aa00ab22d2cefa0d82722bc7245eb3a7f17a8729224b8af8581895e4556a58ac3a323f5a
-  data.tar.gz: 8b8845065dc989f610e31e72f4daaad7c0806d4163a2c257564bda38f96ce2edcaedfc1204877732da0a6e38e340d25015671fc1edcda5be8910cd9204e67d0f
+  metadata.gz: bce6f30ed8df527f77411295005c41e0fba08f4dfafd261a93f325bcb6bf53cbae95194e5cfcaa47f7ee697d25a520df164bfccbdfbb56e39453b9b8b3e39a0f
+  data.tar.gz: b91c0ccac088c23c741effcb9616d89ca36ee439a184134ef793a92a5fc296224851ff0c13ae899735d22b919da5b3bd61d1faaddbb8feb87abde097fcbc3bf5

data/README.md CHANGED Viewed

@@ -9,14 +9,15 @@
 ## Configuration
-Configuration options are:
+Configuration options for fluent.conf are:
 * `kubernetes_url` - URL to the API server. *This is required*
 * `apiVersion` - API version to use (default: `v1beta3`)
-* `client_cert` - path to a client cert file to authenticate to the API server
-* `client_key` - path to a client key file to authenticate to the API server
 * `ca_file` - path to CA file for Kubernetes server certificate validation
 * `verify_ssl` - validate SSL certificates (default: true)
+* `client_cert` - path to a client cert file to authenticate to the API server
+* `client_key` - path to a client key file to authenticate to the API server
+* `bearer_token_file` - path to a file containing the bearer token to use for authentication
 * `container_name_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the Docker container name. This must used named capture groups for `pod_container_name`, `pod_name` & `namespace` (default: `'\/?[^_]+_(?<pod_container_name>[^\.]+)[^_]+_(?<pod_name>[^_]+)_(?<namespace>[^_]+)'`)
 * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)

data/fluent-plugin-kubernetes_metadata_filter.gemspec CHANGED Viewed

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "0.1.0"
+  gem.version       = "0.2.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb CHANGED Viewed

@@ -30,10 +30,11 @@ module Fluent
     config_param :container_name_to_kubernetes_name_regexp,
                  :string,
                  :default => '\/?[^_]+_(?<pod_container_name>[^\.]+)[^_]+_(?<pod_name>[^_]+)_(?<namespace>[^_]+)'
+    config_param :bearer_token_file, :string, :default => ''
-    def self.get_metadata(pod_name, container_name, namespace)
+    def get_metadata(pod_name, container_name, namespace)
       begin
-        metadata = @@client.get_pod(pod_name, namespace)
+        metadata = @client.get_pod(pod_name, namespace)
         if metadata
           return {
             :uid => metadata['metadata']['uid'],
@@ -60,19 +61,27 @@ module Fluent
       require 'active_support/core_ext/object/blank'
       require 'lru_redux'
-      @@client = Kubeclient::Client.new @kubernetes_url, @apiVersion
+      @client = Kubeclient::Client.new @kubernetes_url, @apiVersion
-      if @client_cert.present? && @client_key.present? && @ca_file.present?
+      @client.ssl_options(
+        client_cert: @client_cert.present? ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
+        client_key: @client_key.present? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
+        ca_file: @ca_file,
+        verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+      )
-        @@client.ssl_options(
-          client_cert: OpenSSL::X509::Certificate.new(File.read(@client_cert)),
-          client_key: OpenSSL::PKey::RSA.new(File.read(@client_key)),
-          ca_file: @ca_file,
-          verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
-        )
+      if @bearer_token_file.present?
+        bearer_token = File.read(@bearer_token_file)
+        RestClient.add_before_execution_proc do |req, params|
+          req['authorization'] ||= "Bearer #{bearer_token}"
+        end
       end
-      raise Fluent::ConfigError, 'Invalid Kubernetes API endpoint' unless @@client.api_valid?
+      begin
+        @client.api_valid?
+      rescue KubeException => kube_error
+        raise Fluent::ConfigError, "Invalid Kubernetes API endpoint: #{kube_error.message}"
+      end
       @cache = LruRedux::ThreadSafeCache.new(@cache_size)
       @container_name_to_kubernetes_name_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_name_regexp)
@@ -83,10 +92,11 @@ module Fluent
       es.each {|time, record|
         if record.has_key?(:docker) && record[:docker].has_key?(:name)
+          this = self
           metadata = @cache.getset(record[:docker][:name]){
             match_data = record[:docker][:name].match(@container_name_to_kubernetes_name_regexp_compiled)
             if match_data
-              KubernetesMetadataFilter.get_metadata(
+              this.get_metadata(
                 match_data[:pod_name],
                 match_data[:pod_container_name],
                 match_data[:namespace]
@@ -104,4 +114,4 @@ module Fluent
     end
   end
-end
+end

data/test/cassettes/invalid_api_server_config.yml ADDED Viewed

@@ -0,0 +1,53 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+      Authorization:
+      - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWF
+  response:
+    status:
+      code: 401
+      message: Unauthorized
+    headers:
+      Content-Type:
+      - text/plain; charset=utf-8
+      Date:
+      - Sat, 09 May 2015 14:04:39 GMT
+      Content-Length:
+      - '13'
+    body:
+      encoding: UTF-8
+      string: |
+        Unauthorized
+    http_version:
+  recorded_at: Sat, 09 May 2015 14:04:39 GMT
+recorded_with: VCR 2.9.3

data/test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml ADDED Viewed

@@ -0,0 +1,202 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+      Authorization:
+      - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Sat, 09 May 2015 13:51:07 GMT
+      Content-Length:
+      - '67'
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "versions": [
+            "v1beta1",
+            "v1beta2",
+            "v1beta3"
+          ]
+        }
+    http_version:
+  recorded_at: Sat, 09 May 2015 13:51:07 GMT
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1beta3/namespaces/default/pods/fabric8-console-controller-98rqc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+      Authorization:
+      - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Sat, 09 May 2015 13:51:07 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Pod",
+          "apiVersion": "v1beta3",
+          "metadata": {
+            "name": "fabric8-console-controller-98rqc",
+            "generateName": "fabric8-console-controller-",
+            "namespace": "default",
+            "selfLink": "/api/v1beta3/namespaces/default/pods/fabric8-console-controller-98rqc",
+            "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
+            "resourceVersion": "3556",
+            "creationTimestamp": "2015-05-08T09:22:42Z",
+            "labels": {
+              "component": "fabric8Console"
+            }
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "openshift-cert-secrets",
+                "hostPath": null,
+                "emptyDir": null,
+                "gcePersistentDisk": null,
+                "gitRepo": null,
+                "secret": {
+                  "secretName": "openshift-cert-secrets"
+                },
+                "nfs": null,
+                "iscsi": null,
+                "glusterfs": null
+              }
+            ],
+            "containers": [
+              {
+                "name": "fabric8-console-container",
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "ports": [
+                  {
+                    "containerPort": 9090,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "OAUTH_CLIENT_ID",
+                    "value": "fabric8-console"
+                  },
+                  {
+                    "name": "OAUTH_AUTHORIZE_URI",
+                    "value": "https://localhost:8443/oauth/authorize"
+                  }
+                ],
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "openshift-cert-secrets",
+                    "readOnly": true,
+                    "mountPath": "/etc/secret-volume"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent",
+                "capabilities": {}
+              }
+            ],
+            "restartPolicy": "Always",
+            "dnsPolicy": "ClusterFirst",
+            "host": "jimmi-redhat.localnet"
+          },
+          "status": {
+            "phase": "Running",
+            "Condition": [
+              {
+                "type": "Ready",
+                "status": "True"
+              }
+            ],
+            "hostIP": "172.17.42.1",
+            "podIP": "172.17.0.4",
+            "containerStatuses": [
+              {
+                "name": "fabric8-console-container",
+                "state": {
+                  "running": {
+                    "startedAt": "2015-05-09T13:33:38Z"
+                  }
+                },
+                "lastState": {},
+                "ready": true,
+                "restartCount": 2,
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
+                "containerID": "docker://1b1d1f61c1205fe73328c75b2945e2ce05acfba2fde16299a8103fb22e9ec58a"
+              },
+              {
+                "name": "POD",
+                "state": {
+                  "termination": {
+                    "exitCode": 0,
+                    "startedAt": "2015-05-09T09:45:08Z",
+                    "finishedAt": "2015-05-09T09:47:00Z"
+                  }
+                },
+                "lastState": {},
+                "ready": false,
+                "restartCount": 1,
+                "image": "openshift/origin-pod:v0.4.4",
+                "imageID": "docker://81fa37a83ec46fed9dddf30a9c3c43f54bfe4b65dea74a4c152fd515a0bf92a8",
+                "containerID": "docker://9c8ed1a9bbb6d9804b12379654461dcd3b447438c02c52ca7f7d2504e14ae96b"
+              }
+            ]
+          }
+        }
+    http_version:
+  recorded_at: Sat, 09 May 2015 13:51:07 GMT
+recorded_with: VCR 2.9.3

data/test/plugin/test.token ADDED Viewed

	@@ -0,0 +1 @@
1	+ 12345

data/test/plugin/test_filter_kubernetes_metadata.rb CHANGED Viewed

@@ -61,6 +61,18 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         assert_equal(1, d.instance.cache_size)
       end
     end
+    test 'invalid API server config' do
+      VCR.use_cassette('invalid_api_server_config') do
+        assert_raise Fluent::ConfigError do
+          d = create_driver(%[
+            kubernetes_url https://localhost:8443
+            bearer_token_file test/plugin/test.token
+            verify_ssl false
+          ])
+        end
+      end
+    end
   end
   sub_test_case 'filter_stream' do
@@ -99,6 +111,34 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       end
     end
+    test 'with docker & kubernetes metadata using bearer token' do
+      VCR.use_cassette('kubernetes_docker_metadata_using_bearer_token') do
+        msg = {
+            :docker => {
+                :name => '/k8s_fabric8-console-container.efbd6e64_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_42cbc279'
+            }
+        }
+        es = emit(msg, %[
+          kubernetes_url https://localhost:8443
+          verify_ssl false
+          bearer_token_file test/plugin/test.token
+        ])
+        expected_kube_metadata = {
+            :kubernetes => {
+              :host => "jimmi-redhat.localnet",
+              :pod_name =>"fabric8-console-controller-98rqc",
+              :container_name => "fabric8-console-container",
+              :namespace => "default",
+              :uid => "c76927af-f563-11e4-b32d-54ee7527188d",
+              :labels => {
+                :component => "fabric8Console"
+              }
+            }
+        }
+        assert_equal(msg.merge(expected_kube_metadata), es.instance_variable_get(:@record_array)[0])
+      end
+    end
     test 'with docker metadata, non-kubernetes' do
       VCR.use_cassette('non_kubernetes_docker_metadata') do
         msg = {

metadata CHANGED Viewed

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-08 00:00:00.000000000 Z
+date: 2015-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -208,10 +208,13 @@ files:
 - circle.yml
 - fluent-plugin-kubernetes_metadata_filter.gemspec
 - lib/fluent/plugin/filter_kubernetes_metadata.rb
+- test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata.yml
+- test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
 - test/cassettes/non_kubernetes_docker_metadata.yml
 - test/cassettes/valid_kubernetes_api_server.yml
 - test/helper.rb
+- test/plugin/test.token
 - test/plugin/test_filter_kubernetes_metadata.rb
 homepage: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
 licenses:
@@ -238,8 +241,11 @@ signing_key:
 specification_version: 4
 summary: Filter plugin to add Kubernetes metadata
 test_files:
+- test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata.yml
+- test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
 - test/cassettes/non_kubernetes_docker_metadata.yml
 - test/cassettes/valid_kubernetes_api_server.yml
 - test/helper.rb
+- test/plugin/test.token
 - test/plugin/test_filter_kubernetes_metadata.rb