fluent-plugin-kubernetes-objects 1.1.2 → 1.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4aae816406243c2739dc9b849522ca0fd8fed9e58e17df93de18e748b11b6758
-  data.tar.gz: da9469b542011a7b1f26572862d72ba94683f511458dcca47c083d95e9b2960f
+  metadata.gz: 68aac663e52efbd624bb1f17d99ffd0ef02c93c238c4d1172914d07eaa45d1a5
+  data.tar.gz: 76b6f78a9a60a172d919e31dec6430379993c4991d03cba54221275816c21233
 SHA512:
-  metadata.gz: 6c390d0544d347cfce5c8a06e374ccd27d55d61dba8785069505615ecccf6b22bc2c1063aa2aaba4f3c2ca4d98c160a837e6939fd62333503f9ff3591b5c72cd
-  data.tar.gz: 464880a76e0e38103e71587ad40e1dde8629183e458a16096508b07c8b7ec8a49f6b000ee617c2f6759983ba9d46fc2dab8383e77a88e2e57ae87c1e9a498162
+  metadata.gz: 3cd55f6524fa3ca47d053085f912e3e8a5a7fe76220fe48132882ecb7867f982d9998be1b06e569a39c40677d12c359cbec75d4fa3cbdbdce50ac70e169c2117
+  data.tar.gz: 2420f3c1de9ef618ebaf4acf1e7805d17b93570c09c516e9b2d2fddd515b2b7729cc7034fbae9729b9cf7854f3b4c02753a104b3df37776780befb0e5b06e94d

data/Gemfile.lock

@@ -1,86 +1,96 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes-objects (1.1.1)
-      fluentd (~> 1.4)
-      kubeclient (~> 4.2)
+    fluent-plugin-kubernetes-objects (1.1.6)
+      fluentd (>= 1.9.1)
+      http_parser.rb (= 0.5.3)
+      kubeclient (~> 4.6.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.6.0)
-      public_suffix (>= 2.0.2, < 4.0)
-    cool.io (1.5.3)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
-    dig_rb (1.0.1)
-    docile (1.3.1)
-    domain_name (0.5.20180417)
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    concurrent-ruby (1.1.9)
+    cool.io (1.7.1)
+    crack (0.4.5)
+      rexml
+    docile (1.4.0)
+    domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    fluentd (1.4.1)
+    ffi (1.15.3)
+    ffi-compiler (1.0.1)
+      ffi (>= 1.0.0)
+      rake
+    fluentd (1.13.2)
+      bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      dig_rb (~> 1.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 0.7.0, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
+      http_parser.rb (>= 0.5.1, < 0.8.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (~> 1.0)
+      tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
+      webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
-    hashdiff (0.3.8)
-    http (3.3.0)
+    hashdiff (1.0.1)
+    http (4.4.1)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
-      http-form_data (~> 2.0)
-      http_parser.rb (~> 0.6.0)
-    http-cookie (1.0.3)
+      http-form_data (~> 2.2)
+      http-parser (~> 1.2.0)
+    http-accept (1.7.0)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
-    http-form_data (2.1.1)
-    http_parser.rb (0.6.0)
-    json (2.1.0)
-    kubeclient (4.3.0)
-      http (~> 3.0)
+    http-form_data (2.3.0)
+    http-parser (1.2.3)
+      ffi-compiler (>= 1.0, < 2.0)
+    http_parser.rb (0.5.3)
+    kubeclient (4.6.0)
+      http (>= 3.0, < 5.0)
       recursive-open-struct (~> 1.0, >= 1.0.4)
       rest-client (~> 2.0)
-    mime-types (3.2.2)
+    mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2018.0812)
-    minitest (5.11.3)
-    msgpack (1.2.9)
+    mime-types-data (3.2021.0704)
+    minitest (5.14.4)
+    msgpack (1.4.2)
     netrc (0.11.0)
-    power_assert (1.1.3)
-    public_suffix (3.0.3)
-    rake (12.3.2)
-    recursive-open-struct (1.1.0)
-    rest-client (2.0.2)
+    power_assert (2.0.0)
+    public_suffix (4.0.6)
+    rake (13.0.6)
+    recursive-open-struct (1.1.3)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    safe_yaml (1.0.4)
-    serverengine (2.1.0)
+    rexml (3.2.5)
+    serverengine (2.2.4)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    simplecov (0.16.1)
+    simplecov (0.21.2)
       docile (~> 1.1)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
-    strptime (0.2.3)
-    test-unit (3.3.0)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    strptime (0.2.5)
+    test-unit (3.4.4)
       power_assert
-    thread_safe (0.3.6)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2019.1)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2021.1)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.5)
-    webmock (3.5.1)
+    unf_ext (0.0.7.7)
+    webmock (3.13.0)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
-      hashdiff
+      hashdiff (>= 0.4.0, < 2.0.0)
+    webrick (1.7.0)
     yajl-ruby (1.4.1)
 
 PLATFORMS
@@ -90,10 +100,10 @@ DEPENDENCIES
   bundler (~> 2.0)
   fluent-plugin-kubernetes-objects!
   minitest (~> 5.11)
-  rake (~> 12.3)
+  rake (>= 12.0)
   simplecov
   test-unit (~> 3.3)
   webmock (~> 3.5)
 
 BUNDLED WITH
-   2.0.1
+   2.2.23

data/VERSION

@@ -0,0 +1 @@
+1.1.6

data/fluent-plugin-kubernetes-objects.gemspec

@@ -9,35 +9,25 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/splunk/fluent-plugin-kubernetes-objects'
   spec.license       = 'Apache-2.0'
 
-  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
-  # to allow pushing to a single host or delete this section to allow pushing to any host.
-  # if spec.respond_to?(:metadata)
-  #   spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
-  # else
-  #   raise 'RubyGems 2.0 or newer is required to protect against ' \
-  #     'public gem pushes.'
-  # end
-
   spec.require_paths = ['lib']
   spec.test_files    = Dir.glob('test/**/**.rb')
   spec.files         = %w[
     CODE_OF_CONDUCT.md README.md LICENSE
     fluent-plugin-kubernetes-objects.gemspec
     Gemfile Gemfile.lock
-    Rakefile
+    Rakefile VERSION
   ] + Dir.glob('lib/**/**').reject(&File.method(:directory?))
 
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-
   spec.required_ruby_version = '>= 2.3.0'
-
-  spec.add_runtime_dependency 'fluentd', '~> 1.4'
-  spec.add_runtime_dependency 'kubeclient', '~> 4.2'
+  spec.add_runtime_dependency 'fluentd', '>= 1.9.1'
+  spec.add_runtime_dependency 'kubeclient', '~> 4.6.0'
+  spec.add_runtime_dependency 'http_parser.rb', '= 0.5.3'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'minitest', '~> 5.11'
-  spec.add_development_dependency 'rake', '~> 12.3'
+  spec.add_development_dependency 'rake', '>= 12.0'
   spec.add_development_dependency 'test-unit', '~> 3.3' # required by fluent/test.rb
   spec.add_development_dependency 'webmock', '~> 3.5'
 end

data/lib/fluent/plugin/in_kubernetes_objects.rb

@@ -102,7 +102,6 @@ module Fluent::Plugin
     end
 
     def close
-      @watchers.each &:finish if @watchers
       super
     end
 
@@ -118,6 +117,21 @@ module Fluent::Plugin
       [@tag_prefix, item_name, @tag_suffix].join
     end
 
+    def init_with_kubeconfig()
+      options = {}
+      config = Kubeclient::Config.read @kubeconfig
+      current_context = config.context
+
+      @client = Kubeclient::Client.new(
+        current_context.api_endpoint,
+        current_context.api_version,
+        options.merge(
+          ssl_options: current_context.ssl_options,
+          auth_options: current_context.auth_options
+        )
+      )
+    end
+
     def initialize_client
       # mostly borrowed from Fluentd Kubernetes Metadata Filter Plugin
       if @kubernetes_url.nil?
@@ -173,16 +187,7 @@ module Fluent::Plugin
     end
 
     def start_watchers
-      @watchers = @watch_objects.map do |o|
-        o = o.to_h.dup
-        o[:as] = :raw
-        resource_name = o.delete(:resource_name)
-        watch_interval = o.delete(:interval)
-
-        version = @storage.get(resource_name)
-        o[:resource_version] = version if version
-          create_watcher_thread resource_name, o, watch_interval
-      end
+      @watch_objects.each(&method(:create_watcher_thread))
     end
 
     def create_pull_thread(conf)
@@ -225,19 +230,35 @@ module Fluent::Plugin
       end
     end
 
-    def create_watcher_thread(object_name, watcher, interval)
-      thread_create(:"watch_#{object_name}") do
-        @client.public_send("watch_#{object_name}", watcher).tap { |watcher|
-          tag = generate_tag "#{object_name}.watch"
-          watcher.each do |entity|
-            log.trace { "Received new object from watching #{object_name}" }
-            entity = JSON.parse(entity)
-            router.emit tag, Fluent::Engine.now, entity
-            @storage.put object_name, entity['object']['metadata']['resourceVersion']
-            sleep(interval)
+    def create_watcher_thread(conf)
+      options = conf.to_h.dup
+      options[:as] = :raw
+      resource_name = options[:resource_name]
+      version = @storage.get(resource_name)
+      if version
+        options[:resource_version] = version
+      else
+        options[:resource_version] = 0
+      end
+
+      thread_create :"watch_#{resource_name}" do
+        while thread_current_running?
+          @client.public_send("watch_#{resource_name}", options).tap do |watcher|
+            tag = generate_tag "#{resource_name}"
+            watcher.each do |entity|
+              begin
+                entity = JSON.parse(entity)
+                router.emit tag, Fluent::Engine.now, entity
+                options[:resource_version] = entity['object']['metadata']['resourceVersion']
+                @storage.put resource_name, entity['object']['metadata']['resourceVersion']
+              rescue => e
+                log.info "Got exception #{e} parsing entity #{entity}. Resetting watcher."
+              end
+            end
           end
-        }
+        end
       end
     end
   end
 end
+

data/test/fluent/plugin/in_kubernetes_objects_test.rb

@@ -1,4 +1,5 @@
 require "test_helper"
+require 'fluent/plugin/in_kubernetes_objects.rb'
 
 describe Fluent::Plugin::KubernetesObjectsInput do
   include Fluent::Test::Helpers
@@ -109,9 +110,12 @@ describe Fluent::Plugin::KubernetesObjectsInput do
       </watch>
       CONF
 
+      stub_k8s_events params: {resourceVersion: "0"}
+      stub_k8s_events params: {resourceVersion: "6621683"}
+
       d.run expect_emits: 1, timeout: 3
       events = d.events
-      expect(events.all? { |e| e[0] == 'kubernetes.events.watch'}).must_equal true
+      expect(events.all? { |e| e[0] == 'kubernetes.events'}).must_equal true
     end
 
     it "should use checkpoints for watching" do
@@ -133,6 +137,7 @@ describe Fluent::Plugin::KubernetesObjectsInput do
         CONF
 
         stub_k8s_events params: {resourceVersion: "123456"}
+        stub_k8s_events params: {resourceVersion: "6621683"}
 
         d.run expect_emits: 1, timeout: 3
       ensure

data/test/test_helper.rb

@@ -1,13 +1,12 @@
 require 'simplecov'
 SimpleCov.start
 
-$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+$LOAD_PATH.unshift(File.expand_path('..', __dir__))
 
 # suppress warning, when require the 'http' library shows circle require warning
 # which is pretty annoying (kubeclient depends on http for watch_stream)
 _verbose = $VERBOSE
 $VERBOSE = nil
-require 'fluent/plugin/in_kubernetes_objects'
 $VERBOSE = _verbose
 
 require 'fluent/test'

metadata

@@ -1,43 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes-objects
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.6
 platform: ruby
 authors:
 - Splunk Inc.
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-07-23 00:00:00.000000000 Z
+date: 2021-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.9.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.9.1
 - !ruby/object:Gem::Dependency
   name: kubeclient
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: 4.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: 4.6.0
+- !ruby/object:Gem::Dependency
+  name: http_parser.rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.5.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.5.3
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -70,16 +84,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +136,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- VERSION
 - fluent-plugin-kubernetes-objects.gemspec
 - lib/fluent/plugin/in_kubernetes_objects.rb
 - test/fluent/plugin/in_kubernetes_objects_test.rb
@@ -137,7 +152,7 @@ homepage: https://github.com/splunk/fluent-plugin-kubernetes-objects
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -152,8 +167,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Fluentd Plugin for Kubernetes Objects.
 test_files: