fluent-plugin-kubernetes-objects 1.1.2 → 1.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4aae816406243c2739dc9b849522ca0fd8fed9e58e17df93de18e748b11b6758
-  data.tar.gz: da9469b542011a7b1f26572862d72ba94683f511458dcca47c083d95e9b2960f
+  metadata.gz: 483397cd1c9ccdc7009f104b3c6e8ef94ecf5f56822406e862980301bb846d64
+  data.tar.gz: 523e83eda243fc7d3faf5d2ef9fac4cf9540e84f814b33d03d2f2c93006df4cb
 SHA512:
-  metadata.gz: 6c390d0544d347cfce5c8a06e374ccd27d55d61dba8785069505615ecccf6b22bc2c1063aa2aaba4f3c2ca4d98c160a837e6939fd62333503f9ff3591b5c72cd
-  data.tar.gz: 464880a76e0e38103e71587ad40e1dde8629183e458a16096508b07c8b7ec8a49f6b000ee617c2f6759983ba9d46fc2dab8383e77a88e2e57ae87c1e9a498162
+  metadata.gz: 9c6502046925124ec272b53251d3e08fcb6c0ee69a45f35f7ba87618dac86e4fa7f5665f6340f529d2bd207418314151715df0cc29b24bf0e894ce43c3c0a0b5
+  data.tar.gz: d58d7d3e507c8f77597751868918f04a3fc624785d78956b7deeeaaa7491dabfd4175bf8f331ea2e82a72c93e2e36fbda25185338c09632f37b659debb80f809

data/Gemfile.lock

@@ -1,64 +1,72 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes-objects (1.1.1)
-      fluentd (~> 1.4)
-      kubeclient (~> 4.2)
+    fluent-plugin-kubernetes-objects (1.1.3)
+      fluentd (~> 1.9.1)
+      http_parser.rb (= 0.5.3)
+      kubeclient (~> 4.6.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     addressable (2.6.0)
       public_suffix (>= 2.0.2, < 4.0)
-    cool.io (1.5.3)
+    concurrent-ruby (1.1.6)
+    cool.io (1.6.0)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    dig_rb (1.0.1)
     docile (1.3.1)
-    domain_name (0.5.20180417)
+    domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    fluentd (1.4.1)
+    ffi (1.12.2)
+    ffi-compiler (1.0.1)
+      ffi (>= 1.0.0)
+      rake
+    fluentd (1.9.2)
       cool.io (>= 1.4.5, < 2.0.0)
-      dig_rb (~> 1.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 0.7.0, < 2.0.0)
+      msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.0.4, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (~> 1.0)
+      tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
     hashdiff (0.3.8)
-    http (3.3.0)
+    http (4.3.0)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
-      http-form_data (~> 2.0)
-      http_parser.rb (~> 0.6.0)
+      http-form_data (~> 2.2)
+      http-parser (~> 1.2.0)
+    http-accept (1.7.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    http-form_data (2.1.1)
-    http_parser.rb (0.6.0)
+    http-form_data (2.2.0)
+    http-parser (1.2.1)
+      ffi-compiler (>= 1.0, < 2.0)
+    http_parser.rb (0.5.3)
     json (2.1.0)
-    kubeclient (4.3.0)
-      http (~> 3.0)
+    kubeclient (4.6.0)
+      http (>= 3.0, < 5.0)
       recursive-open-struct (~> 1.0, >= 1.0.4)
       rest-client (~> 2.0)
-    mime-types (3.2.2)
+    mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2018.0812)
+    mime-types-data (3.2019.1009)
     minitest (5.11.3)
-    msgpack (1.2.9)
+    msgpack (1.3.3)
     netrc (0.11.0)
     power_assert (1.1.3)
     public_suffix (3.0.3)
     rake (12.3.2)
     recursive-open-struct (1.1.0)
-    rest-client (2.0.2)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     safe_yaml (1.0.4)
-    serverengine (2.1.0)
+    serverengine (2.2.1)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     simplecov (0.16.1)
@@ -69,14 +77,13 @@ GEM
     strptime (0.2.3)
     test-unit (3.3.0)
       power_assert
-    thread_safe (0.3.6)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2019.1)
+    tzinfo (2.0.1)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2019.3)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.5)
+    unf_ext (0.0.7.6)
     webmock (3.5.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
@@ -90,10 +97,10 @@ DEPENDENCIES
   bundler (~> 2.0)
   fluent-plugin-kubernetes-objects!
   minitest (~> 5.11)
-  rake (~> 12.3)
+  rake (~> 12.0)
   simplecov
   test-unit (~> 3.3)
   webmock (~> 3.5)
 
 BUNDLED WITH
-   2.0.1
+   2.1.4

data/VERSION

@@ -0,0 +1 @@
+1.1.3

data/fluent-plugin-kubernetes-objects.gemspec

@@ -24,20 +24,19 @@ Gem::Specification.new do |spec|
     CODE_OF_CONDUCT.md README.md LICENSE
     fluent-plugin-kubernetes-objects.gemspec
     Gemfile Gemfile.lock
-    Rakefile
+    Rakefile VERSION
   ] + Dir.glob('lib/**/**').reject(&File.method(:directory?))
 
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-
   spec.required_ruby_version = '>= 2.3.0'
-
-  spec.add_runtime_dependency 'fluentd', '~> 1.4'
-  spec.add_runtime_dependency 'kubeclient', '~> 4.2'
+  spec.add_runtime_dependency 'fluentd', '~> 1.9.1'
+  spec.add_runtime_dependency 'kubeclient', '~> 4.6.0'
+  spec.add_runtime_dependency 'http_parser.rb', '= 0.5.3'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'minitest', '~> 5.11'
-  spec.add_development_dependency 'rake', '~> 12.3'
+  spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'test-unit', '~> 3.3' # required by fluent/test.rb
   spec.add_development_dependency 'webmock', '~> 3.5'
 end

data/lib/fluent/plugin/in_kubernetes_objects.rb

@@ -102,7 +102,6 @@ module Fluent::Plugin
     end
 
     def close
-      @watchers.each &:finish if @watchers
       super
     end
 
@@ -118,6 +117,21 @@ module Fluent::Plugin
       [@tag_prefix, item_name, @tag_suffix].join
     end
 
+    def init_with_kubeconfig()
+      options = {}
+      config = Kubeclient::Config.read @kubeconfig
+      current_context = config.context
+
+      @client = Kubeclient::Client.new(
+        current_context.api_endpoint,
+        current_context.api_version,
+        options.merge(
+          ssl_options: current_context.ssl_options,
+          auth_options: current_context.auth_options
+        )
+      )
+    end
+
     def initialize_client
       # mostly borrowed from Fluentd Kubernetes Metadata Filter Plugin
       if @kubernetes_url.nil?
@@ -173,16 +187,7 @@ module Fluent::Plugin
     end
 
     def start_watchers
-      @watchers = @watch_objects.map do |o|
-        o = o.to_h.dup
-        o[:as] = :raw
-        resource_name = o.delete(:resource_name)
-        watch_interval = o.delete(:interval)
-
-        version = @storage.get(resource_name)
-        o[:resource_version] = version if version
-          create_watcher_thread resource_name, o, watch_interval
-      end
+      @watch_objects.each(&method(:create_watcher_thread))
     end
 
     def create_pull_thread(conf)
@@ -225,19 +230,35 @@ module Fluent::Plugin
       end
     end
 
-    def create_watcher_thread(object_name, watcher, interval)
-      thread_create(:"watch_#{object_name}") do
-        @client.public_send("watch_#{object_name}", watcher).tap { |watcher|
-          tag = generate_tag "#{object_name}.watch"
-          watcher.each do |entity|
-            log.trace { "Received new object from watching #{object_name}" }
-            entity = JSON.parse(entity)
-            router.emit tag, Fluent::Engine.now, entity
-            @storage.put object_name, entity['object']['metadata']['resourceVersion']
-            sleep(interval)
+    def create_watcher_thread(conf)
+      options = conf.to_h.dup
+      options[:as] = :raw
+      resource_name = options[:resource_name]
+      version = @storage.get(resource_name)
+      if version
+        options[:resource_version] = version
+      else
+        options[:resource_version] = 0
+      end
+
+      thread_create :"watch_#{resource_name}" do
+        while thread_current_running?
+          @client.public_send("watch_#{resource_name}", options).tap do |watcher|
+            tag = generate_tag "#{resource_name}"
+            watcher.each do |entity|
+              begin
+                entity = JSON.parse(entity)
+                router.emit tag, Fluent::Engine.now, entity
+                options[:resource_version] = entity['object']['metadata']['resourceVersion']
+                @storage.put resource_name, entity['object']['metadata']['resourceVersion']
+              rescue => e
+                log.info "Got exception #{e} parsing entity #{entity}. Resetting watcher."
+              end
+            end
           end
-        }
+        end
       end
     end
   end
 end
+

data/test/fluent/plugin/in_kubernetes_objects_test.rb

@@ -1,4 +1,5 @@
 require "test_helper"
+require 'fluent/plugin/in_kubernetes_objects.rb'
 
 describe Fluent::Plugin::KubernetesObjectsInput do
   include Fluent::Test::Helpers
@@ -109,9 +110,12 @@ describe Fluent::Plugin::KubernetesObjectsInput do
       </watch>
       CONF
 
+      stub_k8s_events params: {resourceVersion: "0"}
+      stub_k8s_events params: {resourceVersion: "6621683"}
+
       d.run expect_emits: 1, timeout: 3
       events = d.events
-      expect(events.all? { |e| e[0] == 'kubernetes.events.watch'}).must_equal true
+      expect(events.all? { |e| e[0] == 'kubernetes.events'}).must_equal true
     end
 
     it "should use checkpoints for watching" do
@@ -133,6 +137,7 @@ describe Fluent::Plugin::KubernetesObjectsInput do
         CONF
 
         stub_k8s_events params: {resourceVersion: "123456"}
+        stub_k8s_events params: {resourceVersion: "6621683"}
 
         d.run expect_emits: 1, timeout: 3
       ensure

data/test/test_helper.rb

@@ -1,13 +1,12 @@
 require 'simplecov'
 SimpleCov.start
 
-$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+$LOAD_PATH.unshift(File.expand_path('..', __dir__))
 
 # suppress warning, when require the 'http' library shows circle require warning
 # which is pretty annoying (kubeclient depends on http for watch_stream)
 _verbose = $VERBOSE
 $VERBOSE = nil
-require 'fluent/plugin/in_kubernetes_objects'
 $VERBOSE = _verbose
 
 require 'fluent/test'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes-objects
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.3
 platform: ruby
 authors:
 - Splunk Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-07-23 00:00:00.000000000 Z
+date: 2020-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,28 +16,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.9.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.9.1
 - !ruby/object:Gem::Dependency
   name: kubeclient
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: 4.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: 4.6.0
+- !ruby/object:Gem::Dependency
+  name: http_parser.rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.5.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.5.3
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +136,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- VERSION
 - fluent-plugin-kubernetes-objects.gemspec
 - lib/fluent/plugin/in_kubernetes_objects.rb
 - test/fluent/plugin/in_kubernetes_objects_test.rb
@@ -152,7 +167,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.2
+rubygems_version: 3.1.1
 signing_key: 
 specification_version: 4
 summary: Fluentd Plugin for Kubernetes Objects.