fluent-plugin-kubernetes_metadata_filter 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d2300a4324a73e67a50f1151cbbfb89beb7f3091
-  data.tar.gz: 95aa9acea4987f1e167e6c0975ad1fff2907b6a1
+  metadata.gz: 7c85f8cb8e0e93bb71aaafe430cbeacedb7877a8
+  data.tar.gz: e7148afb3e76d346a5b1bda9a5952021682b71cb
 SHA512:
-  metadata.gz: 75d9e411f23df7047576e2cb93cf8c025af632235c41b9491ffed5cc8434361603ed654dd1cc605a4c2bd8c7cdc29edf750c9ab0902519a1db317c1ff4701cb0
-  data.tar.gz: f01249b3c1d1e6fe99e15f9a87a1a00060b98d1a78333976d3a1eb7af891f6e7a77dbe433a5226966d9b30456599e53e77cce39d72eb881be0ae381b511a6323
+  metadata.gz: 06fe87cf491110b44c962ce7f092b4dd55aaeb90db9d229c37f673ccdfd981a38af47670feb2eec7749521a149e0512bfa485340d0513ca084b477ea565869de
+  data.tar.gz: b477f9416509a823f2a400d67784c6dc570158088a21cf65c27847e4db1495dd2e730d6f942694ce95d53844d11026abac7aa69b77ed3bdce86664f7528b5a36

data/Gemfile

@@ -1,6 +1,7 @@
 source 'https://rubygems.org'
 
-gem "codeclimate-test-reporter", group: :test, require: nil
+gem 'codeclimate-test-reporter', :group => :test, :require => nil
+gem 'rubocop', require: false
 
 # Specify your gem's dependencies in fluent-plugin-add.gemspec
 gemspec

data/Rakefile

@@ -1,6 +1,6 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
-require "bump/tasks"
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require 'bump/tasks'
 
 task :test => [:base_test]
 
@@ -11,13 +11,13 @@ Rake::TestTask.new(:base_test) do |t|
   # To run test for only one file (or file path pattern)
   #  $ bundle exec rake base_test TEST=test/test_specified_path.rb
   #  $ bundle exec rake base_test TEST=test/test_*.rb
-  t.libs << "test"
-  t.test_files = Dir["test/**/test_*.rb"].sort
+  t.libs << 'test'
+  t.test_files = Dir['test/**/test_*.rb'].sort
   t.verbose = true
   #t.warning = true
 end
 
-desc "Add copyright headers"
+desc 'Add copyright headers'
 task :headers do
   require 'rubygems'
   require 'copyright_header'
@@ -25,7 +25,7 @@ task :headers do
   args = {
     :license => 'ASL2',
     :copyright_software => 'Fluentd Kubernetes Metadata Filter Plugin',
-    :copyright_software_description => "Enrich Fluentd events with Kubernetes metadata",
+    :copyright_software_description => 'Enrich Fluentd events with Kubernetes metadata',
     :copyright_holders => ['Red Hat, Inc.'],
     :copyright_years => ['2015'],
     :add_path => 'lib:test',

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "0.3.0"
+  gem.version       = "0.4.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}
@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency "fluentd"
   gem.add_runtime_dependency "lru_redux"
-  gem.add_runtime_dependency "kubeclient"
+  gem.add_runtime_dependency "kubeclient", "~> 0.1.15"
   gem.add_runtime_dependency "fluent-plugin-docker_metadata_filter"
 
   gem.add_development_dependency "bundler", "~> 1.3"

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -20,9 +20,10 @@ module Fluent
   class KubernetesMetadataFilter < Fluent::Filter
     Fluent::Plugin.register_filter('kubernetes_metadata', self)
 
+    config_param :kubernetes_url, :string
     config_param :cache_size, :integer, :default => 1000
     config_param :cache_ttl, :integer, :default => 60 * 60
-    config_param :kubernetes_url, :string
+    config_param :watch, :bool, :default => true
     config_param :apiVersion, :string, :default => 'v1beta3'
     config_param :client_cert, :string, :default => ''
     config_param :client_key, :string, :default => ''
@@ -38,12 +39,12 @@ module Fluent
         metadata = @client.get_pod(pod_name, namespace)
         if metadata
           return {
-            :uid => metadata['metadata']['uid'],
-            :namespace => metadata['metadata']['namespace'],
-            :pod_name => metadata['metadata']['name'],
-            :container_name => container_name,
-            :labels => metadata['metadata']['labels'].to_h,
-            :host => metadata['spec']['host']
+            uid: metadata['metadata']['uid'],
+            namespace: metadata['metadata']['namespace'],
+            pod_name: metadata['metadata']['name'],
+            container_name: container_name,
+            labels: metadata['metadata']['labels'].to_h,
+            host: metadata['spec']['host']
           }
         end
       rescue KubeException
@@ -73,9 +74,7 @@ module Fluent
 
       if @bearer_token_file.present?
         bearer_token = File.read(@bearer_token_file)
-        RestClient.add_before_execution_proc do |req, params|
-          req['authorization'] ||= "Bearer #{bearer_token}"
-        end
+        @client.bearer_token(bearer_token)
       end
 
       begin
@@ -89,15 +88,22 @@ module Fluent
       end
       @cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
       @container_name_to_kubernetes_name_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_name_regexp)
+
+      if @watch
+        thread = Thread.new(self) { |this|
+          this.start_watch
+        }
+        thread.abort_on_exception = true
+      end
     end
 
     def filter_stream(tag, es)
       new_es = MultiEventStream.new
 
       es.each {|time, record|
-        if record.has_key?(:docker) && record[:docker].has_key?(:name)
+        if record.has_key?(:docker) && record[:docker].has_key?(:id) && record[:docker].has_key?(:name)
           this = self
-          metadata = @cache.getset(record[:docker][:name]){
+          metadata = @cache.getset(record[:docker][:id]){
             match_data = record[:docker][:name].match(@container_name_to_kubernetes_name_regexp_compiled)
             if match_data
               this.get_metadata(
@@ -116,6 +122,39 @@ module Fluent
 
       new_es
     end
-  end
 
+    def start_watch
+      resource_version = @client.get_pods.resourceVersion
+      watcher = @client.watch_pods(resource_version)
+      watcher.each do |notice|
+        puts notice
+        case notice.type
+          when 'MODIFIED'
+            if notice.object.status.containerStatuses
+              notice.object.status.containerStatuses.each { |container_status|
+                if container_status['containerId']
+                  containerId = container_status['containerId'].sub(/^docker:\/\//, '')
+                  cached = @cache[containerId]
+                  if cached
+                    # Only thing that can be modified is labels
+                    cached[:labels] = v.object.metadata.labels.to_h
+                    @cache[containerId] = cached
+                  end
+                end
+              }
+            end
+          when 'DELETED'
+            if notice.object.status.containerStatuses
+              notice.object.status.containerStatuses.each { |container_status|
+                if container_status['containerId']
+                  @cache.delete(container_status['containerId'].sub(/^docker:\/\//, ''))
+                end
+              }
+            end
+          else
+            # ignoring...
+        end
+      end
+    end
+  end
 end

data/test/helper.rb

@@ -16,7 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-require "codeclimate-test-reporter"
+require 'codeclimate-test-reporter'
 SimpleCov.start do
   formatter SimpleCov::Formatter::MultiFormatter[
     SimpleCov::Formatter::HTMLFormatter,
@@ -34,7 +34,7 @@ require 'webmock/test_unit'
 require 'vcr'
 
 VCR.configure do |config|
-  config.cassette_library_dir = "test/cassettes"
+  config.cassette_library_dir = 'test/cassettes'
   config.hook_into :webmock # or :fakeweb
   config.ignore_hosts 'codeclimate.com'
 end
@@ -55,7 +55,7 @@ def ipv6_enabled?
   require 'socket'
 
   begin
-    TCPServer.open("::1", 0)
+    TCPServer.open('::1', 0)
     true
   rescue
     false

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -43,9 +43,10 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
 
     test 'kubernetes url' do
       VCR.use_cassette('valid_kubernetes_api_server') do
-        d = create_driver(%[
+        d = create_driver('
           kubernetes_url https://localhost:8443
-        ])
+          watch false
+        ')
         assert_equal('https://localhost:8443', d.instance.kubernetes_url)
         assert_equal(1000, d.instance.cache_size)
       end
@@ -53,10 +54,11 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
 
     test 'cache size' do
       VCR.use_cassette('valid_kubernetes_api_server') do
-        d = create_driver(%[
+        d = create_driver('
           kubernetes_url https://localhost:8443
+          watch false
           cache_size 1
-        ])
+        ')
         assert_equal('https://localhost:8443', d.instance.kubernetes_url)
         assert_equal(1, d.instance.cache_size)
       end
@@ -65,11 +67,12 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     test 'invalid API server config' do
       VCR.use_cassette('invalid_api_server_config') do
         assert_raise Fluent::ConfigError do
-          d = create_driver(%[
+          d = create_driver('
             kubernetes_url https://localhost:8443
             bearer_token_file test/plugin/test.token
+            watch false
             verify_ssl false
-          ])
+          ')
         end
       end
     end
@@ -77,10 +80,11 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
 
   sub_test_case 'filter_stream' do
 
-    def emit(msg, config=%[
+    def emit(msg, config='
           kubernetes_url https://localhost:8443
+          watch false
           cache_size 1
-        ])
+        ')
       d = create_driver(config)
       d.run {
         d.emit(msg, @time)
@@ -90,22 +94,23 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     test 'with docker & kubernetes metadata' do
       VCR.use_cassette('kubernetes_docker_metadata') do
         msg = {
-            :docker => {
-                :name => '/k8s_fabric8-console-container.efbd6e64_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_42cbc279'
-            }
+          docker: {
+            id: '/49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
+            name: '/k8s_fabric8-console-container.efbd6e64_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_42cbc279'
+          }
         }
         es = emit(msg)
         expected_kube_metadata = {
-            :kubernetes => {
-              :host => "jimmi-redhat.localnet",
-              :pod_name =>"fabric8-console-controller-98rqc",
-              :container_name => "fabric8-console-container",
-              :namespace => "default",
-              :uid => "c76927af-f563-11e4-b32d-54ee7527188d",
-              :labels => {
-                :component => "fabric8Console"
-              }
+          kubernetes: {
+            host:           'jimmi-redhat.localnet',
+            pod_name:       'fabric8-console-controller-98rqc',
+            container_name: 'fabric8-console-container',
+            namespace:      'default',
+            uid:            'c76927af-f563-11e4-b32d-54ee7527188d',
+            labels: {
+              component: 'fabric8Console'
             }
+          }
         }
         assert_equal(msg.merge(expected_kube_metadata), es.instance_variable_get(:@record_array)[0])
       end
@@ -114,26 +119,28 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     test 'with docker & kubernetes metadata using bearer token' do
       VCR.use_cassette('kubernetes_docker_metadata_using_bearer_token') do
         msg = {
-            :docker => {
-                :name => '/k8s_fabric8-console-container.efbd6e64_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_42cbc279'
-            }
+          docker: {
+            id: '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
+            name: '/k8s_fabric8-console-container.efbd6e64_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_42cbc279'
+          }
         }
-        es = emit(msg, %[
+        es = emit(msg, '
           kubernetes_url https://localhost:8443
           verify_ssl false
+          watch false
           bearer_token_file test/plugin/test.token
-        ])
+        ')
         expected_kube_metadata = {
-            :kubernetes => {
-              :host => "jimmi-redhat.localnet",
-              :pod_name =>"fabric8-console-controller-98rqc",
-              :container_name => "fabric8-console-container",
-              :namespace => "default",
-              :uid => "c76927af-f563-11e4-b32d-54ee7527188d",
-              :labels => {
-                :component => "fabric8Console"
-              }
+          kubernetes: {
+            host:           'jimmi-redhat.localnet',
+            pod_name:       'fabric8-console-controller-98rqc',
+            container_name: 'fabric8-console-container',
+            namespace:      'default',
+            uid:            'c76927af-f563-11e4-b32d-54ee7527188d',
+            labels: {
+              component: 'fabric8Console'
             }
+          }
         }
         assert_equal(msg.merge(expected_kube_metadata), es.instance_variable_get(:@record_array)[0])
       end
@@ -142,9 +149,10 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     test 'with docker metadata, non-kubernetes' do
       VCR.use_cassette('non_kubernetes_docker_metadata') do
         msg = {
-            :docker => {
-                :name => '/k8s_POD.c0b903ca_fabric8-forge-controller-ymkew_default_bcde9961-f4b7-11e4-bdbf-54ee7527188d_e1f00705'
-            }
+          docker: {
+            id: '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
+            name: '/k8s_POD.c0b903ca_fabric8-forge-controller-ymkew_default_bcde9961-f4b7-11e4-bdbf-54ee7527188d_e1f00705'
+          }
         }
         es = emit(msg)
         assert_equal(msg, es.instance_variable_get(:@record_array)[0])

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-10 00:00:00.000000000 Z
+date: 2015-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -42,16 +42,16 @@ dependencies:
   name: kubeclient
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.15
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.15
 - !ruby/object:Gem::Dependency
   name: fluent-plugin-docker_metadata_filter
   requirement: !ruby/object:Gem::Requirement