fluent-plugin-kubernetes-objects 1.1.0 → 1.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6cfb891f22b53744c43fc6c2e1518d84a6b9f777715b8e76cdf6203d060dfebb
-  data.tar.gz: 33e009c355bddf26ab670daf63891344edcba1e25cc449f482fd4021ff9537f9
+  metadata.gz: ebbda5fdcf1ccdfc296bd7de8c270b930241d7aeb270b3a75af166e11613a179
+  data.tar.gz: 0ca3df1c39ad412487c891c4e097b30e9516e07e1ab2deb50f5e887c0c761095
 SHA512:
-  metadata.gz: 16d37fd0ff1778a2f02d60bb08fb45137fd18bacd4484add87df8d7d6fbda0a231225dcc443dc20d93df7ce6aee1bc03e22f08020260eb80af413959f693529d
-  data.tar.gz: 66ccb72b7b271c55d67e573fbb614a57e7c6a4fa1bc8755d2eb47d09af3e5481865219c3e8ee3a291416365f53d5d3341b4cc510f32f4418be8ae9915f7a0aab
+  metadata.gz: 80f55f4be2a6d9177c3633e08d05b10c8ab854f275d724fec0fdcd86ab846ec496b69a3c1ccb6fbc55dcaad74ac598caecd7144577085f49fc66f1e2ed684152
+  data.tar.gz: 511205cdb4eb9cd1d977700fbd9d0fd9b31804ca19257965d59423bffc82200d35cc6fcfe329a560d79964d2fb0adc26bae885585dd734d707d0957ae07cf04e

data/Gemfile.lock

@@ -1,86 +1,94 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes-objects (1.0.0)
-      fluentd (~> 1.3)
-      kubeclient (~> 4.2)
+    fluent-plugin-kubernetes-objects (1.1.5)
+      fluentd (>= 1.9.1)
+      http_parser.rb (= 0.5.3)
+      kubeclient (~> 4.6.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.6.0)
-      public_suffix (>= 2.0.2, < 4.0)
-    cool.io (1.5.3)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
-    dig_rb (1.0.1)
-    docile (1.3.1)
-    domain_name (0.5.20180417)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    concurrent-ruby (1.1.8)
+    cool.io (1.7.1)
+    crack (0.4.5)
+      rexml
+    docile (1.3.5)
+    domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    fluentd (1.3.3)
+    ffi (1.14.2)
+    ffi-compiler (1.0.1)
+      ffi (>= 1.0.0)
+      rake
+    fluentd (1.12.1)
+      bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      dig_rb (~> 1.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 0.7.0, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (~> 1.0)
+      tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
-    hashdiff (0.3.8)
-    http (3.3.0)
+    hashdiff (1.0.1)
+    http (4.4.1)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
-      http-form_data (~> 2.0)
-      http_parser.rb (~> 0.6.0)
+      http-form_data (~> 2.2)
+      http-parser (~> 1.2.0)
+    http-accept (1.7.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    http-form_data (2.1.1)
-    http_parser.rb (0.6.0)
-    json (2.1.0)
-    kubeclient (4.2.2)
-      http (~> 3.0)
+    http-form_data (2.3.0)
+    http-parser (1.2.3)
+      ffi-compiler (>= 1.0, < 2.0)
+    http_parser.rb (0.5.3)
+    kubeclient (4.6.0)
+      http (>= 3.0, < 5.0)
       recursive-open-struct (~> 1.0, >= 1.0.4)
       rest-client (~> 2.0)
-    mime-types (3.2.2)
+    mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2018.0812)
-    minitest (5.11.3)
-    msgpack (1.2.6)
+    mime-types-data (3.2021.0225)
+    minitest (5.14.4)
+    msgpack (1.4.2)
     netrc (0.11.0)
-    power_assert (1.1.3)
-    public_suffix (3.0.3)
-    rake (12.3.2)
-    recursive-open-struct (1.1.0)
-    rest-client (2.0.2)
+    power_assert (2.0.0)
+    public_suffix (4.0.6)
+    rake (13.0.3)
+    recursive-open-struct (1.1.3)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    safe_yaml (1.0.4)
-    serverengine (2.1.0)
+    rexml (3.2.4)
+    serverengine (2.2.3)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    simplecov (0.16.1)
+    simplecov (0.21.2)
       docile (~> 1.1)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
-    strptime (0.2.3)
-    test-unit (3.3.0)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.2)
+    strptime (0.2.5)
+    test-unit (3.4.0)
       power_assert
-    thread_safe (0.3.6)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2018.9)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2021.1)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.5)
-    webmock (3.5.1)
+    unf_ext (0.0.7.7)
+    webmock (3.12.0)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
-      hashdiff
+      hashdiff (>= 0.4.0, < 2.0.0)
     yajl-ruby (1.4.1)
 
 PLATFORMS
@@ -90,10 +98,10 @@ DEPENDENCIES
   bundler (~> 2.0)
   fluent-plugin-kubernetes-objects!
   minitest (~> 5.11)
-  rake (~> 12.3)
+  rake (>= 12.0)
   simplecov
   test-unit (~> 3.3)
   webmock (~> 3.5)
 
 BUNDLED WITH
-   2.0.1
+   2.2.13

data/VERSION

@@ -0,0 +1 @@
+1.1.5

data/fluent-plugin-kubernetes-objects.gemspec

@@ -9,35 +9,25 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/splunk/fluent-plugin-kubernetes-objects'
   spec.license       = 'Apache-2.0'
 
-  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
-  # to allow pushing to a single host or delete this section to allow pushing to any host.
-  #if spec.respond_to?(:metadata)
-  #  spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
-  #else
-  #  raise 'RubyGems 2.0 or newer is required to protect against ' \
-  #    'public gem pushes.'
-  #end
-
   spec.require_paths = ['lib']
   spec.test_files    = Dir.glob('test/**/**.rb')
   spec.files         = %w[
     CODE_OF_CONDUCT.md README.md LICENSE
     fluent-plugin-kubernetes-objects.gemspec
     Gemfile Gemfile.lock
-    Rakefile
+    Rakefile VERSION
   ] + Dir.glob('lib/**/**').reject(&File.method(:directory?))
 
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-
   spec.required_ruby_version = '>= 2.3.0'
-
-  spec.add_runtime_dependency 'fluentd', '~> 1.3'
-  spec.add_runtime_dependency 'kubeclient', '~> 4.2'
+  spec.add_runtime_dependency 'fluentd', '>= 1.9.1'
+  spec.add_runtime_dependency 'kubeclient', '~> 4.6.0'
+  spec.add_runtime_dependency 'http_parser.rb', '= 0.5.3'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'minitest', '~> 5.11'
-  spec.add_development_dependency 'rake', '~> 12.3'
+  spec.add_development_dependency 'rake', '>= 12.0'
   spec.add_development_dependency 'test-unit', '~> 3.3' # required by fluent/test.rb
   spec.add_development_dependency 'webmock', '~> 3.5'
 end

data/lib/fluent/plugin/in_kubernetes_objects.rb

@@ -5,7 +5,7 @@ require 'kubeclient'
 
 module Fluent::Plugin
   class KubernetesObjectsInput < Fluent::Plugin::Input
-    VERSION = '1.1.0'.freeze
+    VERSION = '1.1.2'.freeze
 
     Fluent::Plugin.register_input('kubernetes_objects', self)
 
@@ -72,6 +72,9 @@ module Fluent::Plugin
 
       desc 'A selector to restrict the list of returned objects by fields.'
       config_param :field_selector, :string, default: nil
+
+      desc 'The interval at which the objects will be watched.'
+      config_param :interval, :time, default: 15 * 60
     end
 
     config_section :storage do
@@ -99,7 +102,6 @@ module Fluent::Plugin
     end
 
     def close
-      @watchers.each &:finish if @watchers
       super
     end
 
@@ -115,6 +117,21 @@ module Fluent::Plugin
       [@tag_prefix, item_name, @tag_suffix].join
     end
 
+    def init_with_kubeconfig()
+      options = {}
+      config = Kubeclient::Config.read @kubeconfig
+      current_context = config.context
+
+      @client = Kubeclient::Client.new(
+        current_context.api_endpoint,
+        current_context.api_version,
+        options.merge(
+          ssl_options: current_context.ssl_options,
+          auth_options: current_context.auth_options
+        )
+      )
+    end
+
     def initialize_client
       # mostly borrowed from Fluentd Kubernetes Metadata Filter Plugin
       if @kubernetes_url.nil?
@@ -140,7 +157,7 @@ module Fluent::Plugin
         if @bearer_token_file.nil? && File.exist?(secret_token_file)
           @bearer_token_file = secret_token_file
         end
-    end
+      end
 
       ssl_options = {
         client_cert: @client_cert && OpenSSL::X509::Certificate.new(File.read(@client_cert)),
@@ -170,16 +187,7 @@ module Fluent::Plugin
     end
 
     def start_watchers
-      @watchers = @watch_objects.map do |o|
-        o = o.to_h.dup
-        o[:as] = :raw
-        resource_name = o.delete(:resource_name)
-        version = @storage.get(resource_name)
-        o[:resource_version] = version if version
-        @client.public_send("watch_#{resource_name}", o).tap do |watcher|
-          create_watcher_thread resource_name, watcher
-        end
-      end
+      @watch_objects.each(&method(:create_watcher_thread))
     end
 
     def create_pull_thread(conf)
@@ -209,7 +217,7 @@ module Fluent::Plugin
                         ->(item) { item['metadata'].update requestResourceVersion: resource_version }
                       else
                         ->(item) {}
-          end
+                      end
 
           # result['items'] might be nil due to https://github.com/kubernetes/kubernetes/issues/13096
           items = result['items'].to_a
@@ -222,16 +230,35 @@ module Fluent::Plugin
       end
     end
 
-    def create_watcher_thread(object_name, watcher)
-      thread_create(:"watch_#{object_name}") do
-        tag = generate_tag "#{object_name}.watch"
-        watcher.each do |entity|
-          log.trace { "Received new object from watching #{object_name}" }
-          entity = JSON.parse(entity)
-          router.emit tag, Fluent::Engine.now, entity
-          @storage.put object_name, entity['object']['metadata']['resourceVersion']
+    def create_watcher_thread(conf)
+      options = conf.to_h.dup
+      options[:as] = :raw
+      resource_name = options[:resource_name]
+      version = @storage.get(resource_name)
+      if version
+        options[:resource_version] = version
+      else
+        options[:resource_version] = 0
+      end
+
+      thread_create :"watch_#{resource_name}" do
+        while thread_current_running?
+          @client.public_send("watch_#{resource_name}", options).tap do |watcher|
+            tag = generate_tag "#{resource_name}"
+            watcher.each do |entity|
+              begin
+                entity = JSON.parse(entity)
+                router.emit tag, Fluent::Engine.now, entity
+                options[:resource_version] = entity['object']['metadata']['resourceVersion']
+                @storage.put resource_name, entity['object']['metadata']['resourceVersion']
+              rescue => e
+                log.info "Got exception #{e} parsing entity #{entity}. Resetting watcher."
+              end
+            end
+          end
         end
       end
     end
   end
 end
+

data/test/fluent/plugin/in_kubernetes_objects_test.rb

@@ -1,4 +1,5 @@
 require "test_helper"
+require 'fluent/plugin/in_kubernetes_objects.rb'
 
 describe Fluent::Plugin::KubernetesObjectsInput do
   include Fluent::Test::Helpers
@@ -109,9 +110,12 @@ describe Fluent::Plugin::KubernetesObjectsInput do
       </watch>
       CONF
 
+      stub_k8s_events params: {resourceVersion: "0"}
+      stub_k8s_events params: {resourceVersion: "6621683"}
+
       d.run expect_emits: 1, timeout: 3
       events = d.events
-      expect(events.all? { |e| e[0] == 'kubernetes.events.watch'}).must_equal true
+      expect(events.all? { |e| e[0] == 'kubernetes.events'}).must_equal true
     end
 
     it "should use checkpoints for watching" do
@@ -133,9 +137,10 @@ describe Fluent::Plugin::KubernetesObjectsInput do
         CONF
 
         stub_k8s_events params: {resourceVersion: "123456"}
+        stub_k8s_events params: {resourceVersion: "6621683"}
 
         d.run expect_emits: 1, timeout: 3
-            ensure
+      ensure
         f.unlink
       end
     end

data/test/test_helper.rb

@@ -1,13 +1,12 @@
 require 'simplecov'
 SimpleCov.start
 
-$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+$LOAD_PATH.unshift(File.expand_path('..', __dir__))
 
 # suppress warning, when require the 'http' library shows circle require warning
 # which is pretty annoying (kubeclient depends on http for watch_stream)
 _verbose = $VERBOSE
 $VERBOSE = nil
-require 'fluent/plugin/in_kubernetes_objects'
 $VERBOSE = _verbose
 
 require 'fluent/test'

metadata

@@ -1,43 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes-objects
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.5
 platform: ruby
 authors:
 - Splunk Inc.
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-03-07 00:00:00.000000000 Z
+date: 2021-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 1.9.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 1.9.1
 - !ruby/object:Gem::Dependency
   name: kubeclient
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: 4.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: 4.6.0
+- !ruby/object:Gem::Dependency
+  name: http_parser.rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.5.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.5.3
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -70,16 +84,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +136,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- VERSION
 - fluent-plugin-kubernetes-objects.gemspec
 - lib/fluent/plugin/in_kubernetes_objects.rb
 - test/fluent/plugin/in_kubernetes_objects_test.rb
@@ -137,7 +152,7 @@ homepage: https://github.com/splunk/fluent-plugin-kubernetes-objects
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -152,8 +167,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
-signing_key: 
+rubygems_version: 3.1.1
+signing_key:
 specification_version: 4
 summary: Fluentd Plugin for Kubernetes Objects.
 test_files: