fluent-plugin-kubernetes-objects 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d39161509a5767c7b3bb242c99b750af77fd3d5999b37afc58fc8a450f9f68be
+  data.tar.gz: c87dd9f4fe3c9003cc56f5033d0492eb0f4c558ad2fefe00cb312c7ed8896910
+SHA512:
+  metadata.gz: 030e4ff7c8ed1a20fd6b27b10a8324e07e0e0d47c4ee36438c199a919ff8c71fadb728bd3c50ba91bf23db801acdd0873704d71201ad4c7321db94ee4e98cd79
+  data.tar.gz: 2eaa921a968b667d146b195239e8ed2661d7662116abc6c1388d8a1248abb1c75043dfa4bb3e3cafd431039867e7fb018219cfabc047358fd56ecade3622fbec

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at zliang@splunk.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in fluent-plugin-kubernetes_objects_input.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,91 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-kubernetes-objects (1.0.0)
+      fluentd (~> 1.0)
+      kubeclient (~> 3.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    cool.io (1.5.3)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    dig_rb (1.0.1)
+    domain_name (0.5.20170404)
+      unf (>= 0.0.5, < 1.0.0)
+    fluentd (1.1.0)
+      cool.io (>= 1.4.5, < 2.0.0)
+      dig_rb (~> 1.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 0.7.0, < 2.0.0)
+      serverengine (>= 2.0.4, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (~> 1.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    hashdiff (0.3.7)
+    http (2.2.2)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 1.0.1)
+      http_parser.rb (~> 0.6.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http-form_data (1.0.3)
+    http_parser.rb (0.6.0)
+    kubeclient (3.0.0)
+      http (~> 2.2.2)
+      recursive-open-struct (~> 1.0.4)
+      rest-client (~> 2.0)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    minitest (5.11.3)
+    msgpack (1.2.4)
+    netrc (0.11.0)
+    power_assert (1.1.1)
+    public_suffix (3.0.2)
+    rake (10.5.0)
+    recursive-open-struct (1.0.5)
+    rest-client (2.0.2)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    safe_yaml (1.0.4)
+    serverengine (2.0.6)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.3)
+    test-unit (3.2.7)
+      power_assert
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2018.3)
+      tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.5)
+    webmock (3.3.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+    yajl-ruby (1.3.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  fluent-plugin-kubernetes-objects!
+  minitest (~> 5.0)
+  rake (~> 10.0)
+  test-unit (~> 3.0)
+  webmock (~> 3.2)
+
+BUNDLED WITH
+   1.16.1

data/LICENSE

@@ -0,0 +1,233 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2018 Splunk Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+   
+=======================================================================
+Fluentd Plugin for Kubernetes Objects:
+
+The Fluentd Plugin for Kubernetes Objects project contains
+subcomponents with separate copyright notices and license terms.
+Your use of the source code for the these subcomponents is subject
+to the terms and conditions of the following licenses.
+
+========================================================================
+Apache License 2.0
+========================================================================
+The following components are provided under the Apache License 2.0. See project link for details.
+
+    (Apache License 2.0) fluentd (https://github.com/fluent/fluentd/blob/master/LICENSE)
+
+========================================================================
+MIT licenses
+========================================================================
+The following components are provided under the MIT License. See project link for details.
+
+    (MIT License) kubeclient (https://github.com/abonas/kubeclient/blob/master/LICENSE.txt)
+    (MIT License) bundler (https://github.com/bundler/bundler/blob/master/LICENSE.md)
+    (MIT License) rake (https://github.com/ruby/rake/blob/master/MIT-LICENSE)
+    (MIT License) webmock (https://github.com/bblimke/webmock/blob/master/LICENSE)
+    (MIT License) minitest (https://github.com/seattlerb/minitest/blob/master/README.rdoc#license)
+
+========================================================================
+For test-unit:
+========================================================================
+
+See https://github.com/test-unit/test-unit/blob/master/COPYING

data/README.md

@@ -0,0 +1,169 @@
+[ ![Codeship Status for splunk/fluent-plugin-kubernetes-objects](https://app.codeship.com/projects/8ae91f00-261e-0136-02ab-6e830b67b562/status?branch=master)](https://app.codeship.com/projects/286766)
+# fluent-plugin-kubernetes-objects
+
+[Fluentd](https://fluentd.org/) input plugin to collect [objects](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/) in a kubernetes cluster.
+
+This input plugin can collects data in two ways:
+* pull: it collects all available objects at a interval by calling the list APIs.
+* watch: it collects only the new objects when they show up by calling the watch APIs.
+
+## Installation
+
+See also: [Plugin Management](https://docs.fluentd.org/v1.0/articles/plugin-management).
+
+### RubyGems
+
+```
+$ gem install fluent-plugin-kubernetes-objects
+```
+
+### Bundler
+
+Add following line to your Gemfile:
+
+```ruby
+gem "fluent-plugin-kubernetes-objects"
+```
+
+And then execute:
+
+```
+$ bundle
+```
+
+## Configuration
+
+### Example
+
+```
+<source>
+  @type kubernetes_objects
+  tag k8s.*
+
+  <pull>
+    resource_name nodes
+  </pull>
+
+  <pull>
+    resource_name pods
+    namespace default
+  </pull>
+
+  <watch>
+    resource_name events
+  </watch>
+```
+
+In this example, it will pull all the `nodes` , and all the `pods` in `default` namespace, and also watch `events`. And all those data will be tagged with prefix `"k8s."`.
+
+### Parameters
+
+#### @type (string) (required)
+
+This must be `kubernetes_objects`.
+
+#### tag (string) (optional)
+
+The tag of the event.
+
+`*` can be used as a placeholder that expands to the actual resource name (watch objects will have `.watch` suffix). For example, if you set
+
+```
+tag k8s.*
+<pull>
+  resource_name pods
+</pull>
+<watch>
+  resource_name events
+</watch>
+```
+
+The tag for `pods` data will be `k8s.pods`, while the tag for `events` will be `k8s.events.watch`.
+
+Default value: `kubernetes.*`.
+
+#### kubernetes_url (string) (optional)
+
+The URL to the kubernetes API endpoint. By default, it will read from environment variables `KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT`. If those environment variables are not available, and this parameter is not set, error will be raised.
+
+#### api_version (string) (optional)
+
+Kubernetes API version, it includes the API group name and the version, just like the `apiVersion` in a Kubernetes manifests YAML file. E.g. `v1`, `rbac.authorization.k8s.io/v1`, etc.
+
+Default value: `v1`.
+
+#### client_cert (string) (optional)
+
+Path to the certificate file for this client.
+
+#### client_key (string) (optional)
+
+Path to the private key file for this client.
+
+#### ca_file (string) (optional)
+
+Path to the CA file.
+
+#### insecure_ssl (bool) (optional)
+
+When set to `true`, it will ignore inscure `HTTPS` connections (i.e. it ignores server SSL certificate verification errors).
+
+Default value: `false`.
+
+#### secret_dir (string) (optional)
+
+Path of the location where pod's service account's credentials are stored.th of the location where pod's service account's credentials are stored.
+
+Default value: `/var/run/secrets/kubernetes.io/serviceaccount`.
+
+#### bearer_token_file (string) (optional)
+
+Path to the file contains the API token. By default it reads from the file "token" in the `secret_dir`.
+
+#### &lt;pull&gt; Section (optional) (multiple)
+
+Ths &lt;pull&gt; section is used to define which object to pull from the cluster. One section defines one object.
+
+##### resource_name (string) (required)
+
+The name of the object to pull. E.g. `pods`, `nodes`. This must match `api_version`.
+The `resource_name` is not exactly the same as the name in Kubernetes API.
+If a name contains multiple words, e.g. `daemonsets` and `replicasets`,
+you will need use underscore `_` to separate the words.
+So `daemonsets` becomes `daemon_sets`, and `replicasets` becomes `replica_sets`, and so on.
+
+##### namespace (string) (optional)
+
+Only the objects belong to the namespace specified in this parameter will be pulled. When it's not set, it will pull from all name spaces.
+
+#### label_selector (string) (optional)
+
+A selector to restrict the list of returned objects by labels.
+
+#### field_selector (string) (optional)
+
+A selector to restrict the list of returned objects by fields.
+
+#### interval (time) (optional)
+
+The interval at which the objects will be pulled.
+
+Default value: `15m`.
+
+#### &lt;watch&gt; Section (optional) (multiple)
+
+Ths &lt;watch&gt; section is used to define which object to watch from the cluster. One section defines one object.
+
+This section has exactly the same parameters except `interval` as the &lt;pull&gt; section does. Check the &lt;pull&gt; section above for details.
+
+#### &lt;storage&gt; Section (optional) (single)
+
+Defines where to storage checkpoints for `watch`. Each object the plugin watches, it will record the latest `resoruce_version` of that object. And when the fluentd restarts, the plugin will send the previously recorded `resource_version` to the kubernetes watch API.
+
+By default, it uses the memory storage.
+
+See also [Storage section configurations](https://docs.fluentd.org/v1.0/articles/storage-section) and [Storage Plugin Overview](https://docs.fluentd.org/v1.0/articles/storage-plugin-overview).
+
+## License
+
+See [License](LICENSE).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/fluent-plugin-kubernetes-objects.gemspec

@@ -0,0 +1,43 @@
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-kubernetes-objects"
+  spec.version       = "1.0.0"
+  spec.authors       = ["Gimi Liang"]
+  spec.email         = ["zliang@splunk.com"]
+
+  spec.summary       = %q{Fluentd Plugin for Kubernetes Objects.}
+  spec.description   = %q{A Fluentd input plugin for collecting Kubernetes objects, e.g. pods, namespaces, events, etc. by pulling or watching.}
+  spec.homepage      = "https://github.com/splunk/fluent-plugin-kubernetes-objects"
+  spec.license       = "Apache-2.0"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  # if spec.respond_to?(:metadata)
+  #   spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+  # else
+  #   raise "RubyGems 2.0 or newer is required to protect against " \
+  #     "public gem pushes."
+  # end
+
+  spec.require_paths = ["lib"]
+  spec.test_files    = Dir.glob('test/**/**.rb')
+  spec.files         = %w[
+    CODE_OF_CONDUCT.md README.md LICENSE
+    fluent-plugin-kubernetes-objects.gemspec
+    Gemfile Gemfile.lock
+    Rakefile
+  ] + Dir.glob('lib/**/**').reject(&File.method(:directory?))
+
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+
+  spec.required_ruby_version = '>= 2.3.0'
+
+  spec.add_runtime_dependency "fluentd", "~> 1.0"
+  spec.add_runtime_dependency "kubeclient", "~> 3.0"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit", "~> 3.0" # required by fluent/test.rb
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "webmock", "~> 3.2"
+end

data/lib/fluent/plugin/in_kubernetes_objects.rb

@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/input"
+require 'kubeclient'
+
+module Fluent::Plugin
+  class KubernetesObjectsInput < Fluent::Plugin::Input
+    VERSION = "1.0.0"
+
+    Fluent::Plugin.register_input('kubernetes_objects', self)
+
+    helpers :storage, :thread
+
+    desc 'The tag of the event.'
+    config_param :tag, :string, default: 'kubernetes.*'
+
+    desc 'URL of the kubernetes API.'
+    config_param :kubernetes_url, :string, default: nil
+
+    desc 'Kubernetes API version.'
+    config_param :api_version, :string, default: 'v1'
+
+    desc 'Path to the certificate file for this client.'
+    config_param :client_cert, :string, default: nil
+
+    desc 'Path to the private key file for this client.'
+    config_param :client_key, :string, default: nil
+
+    desc 'Path to the CA file.'
+    config_param :ca_file, :string, default: nil
+
+    desc "If `insecure_ssl` is set to `true`, it won't verify apiserver's certificate."
+    config_param :insecure_ssl, :bool, default: false
+
+    desc 'Path to the file contains the API token. By default it reads from the file "token" in the `secret_dir`.'
+    config_param :bearer_token_file, :string, default: nil
+
+    desc "Path of the location where pod's service account's credentials are stored."
+    config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
+
+    desc 'Define a resource to pull.'
+    config_section :pull, required: false, init: false, multi: true, param_name: :pull_objects do
+      desc 'The name of the resource, e.g. "nodes".'
+      config_param :resource_name, :string
+
+      desc 'The namespace of the resource, it pulls from all namespaces if not set.'
+      config_param :namespace, :string, default: nil
+
+      desc 'A selector to restrict the list of returned objects by labels.'
+      config_param :label_selector, :string, default: nil
+
+      desc 'A selector to restrict the list of returned objects by fields.'
+      config_param :field_selector, :string, default: nil
+
+      desc 'The interval at which the objects will be pulled.'
+      config_param :interval, :time, default: 15 * 60
+    end
+
+    desc 'Define a resource to watch.'
+    config_section :watch, required: false, init: false, multi: true, param_name: :watch_objects do
+      desc 'The name of the resource, e.g. "events".'
+      config_param :resource_name, :string
+
+      desc 'The namespace of the resource, it watches all namespaces if not set.'
+      config_param :namespace, :string, default: nil
+
+      desc 'The name of the entity to watch, use this to watch only one entity.'
+      config_param :entity_name, :string, default: nil
+
+      desc 'A selector to restrict the list of returned objects by labels.'
+      config_param :label_selector, :string, default: nil
+
+      desc 'A selector to restrict the list of returned objects by fields.'
+      config_param :field_selector, :string, default: nil
+    end
+
+    config_section :storage do
+      # use memory by default
+      config_set_default :usage, "checkpoints"
+      config_set_default :@type, "local"
+      config_set_default :persistent, false
+    end
+
+    def configure(conf)
+      super
+
+      raise Fluent::ConfigError, "At least one <pull> or <watch> is required, but found none." if @pull_objects.empty? && @watch_objects.empty?
+
+      @storage = storage_create usage: "checkpoints"
+
+      parse_tag
+      initialize_client
+    end
+
+    def start
+      super
+      start_pullers
+      start_watchers
+    end
+
+    def close
+      @watchers.each &:finish if @watchers
+      super
+    end
+
+    private
+
+    def parse_tag
+      @tag_prefix, @tag_suffix = @tag.split('*') if @tag.include?('*')
+    end
+
+    def generate_tag(item_name)
+      return @tag unless @tag_prefix
+
+      [@tag_prefix, item_name, @tag_suffix].join
+    end
+
+    def initialize_client
+      # mostly borrowed from Fluentd Kubernetes Metadata Filter Plugin
+      if @kubernetes_url.nil?
+	# Use Kubernetes default service account if we're in a pod.
+	env_host = ENV['KUBERNETES_SERVICE_HOST']
+	env_port = ENV['KUBERNETES_SERVICE_PORT']
+	if env_host && env_port
+	  @kubernetes_url = "https://#{env_host}:#{env_port}/#{@api_version == 'v1' ? 'api' : 'apis'}"
+	end
+      end
+
+      raise Fluent::ConfigError, "kubernetes url is not set" unless @kubernetes_url
+
+      # Use SSL certificate and bearer token from Kubernetes service account.
+      if Dir.exist?(@secret_dir)
+	secret_ca_file = File.join(@secret_dir, 'ca.cert')
+	secret_token_file = File.join(@secret_dir, 'token')
+
+	if @ca_file.nil? and File.exist?(secret_ca_file)
+	  @ca_file = secret_ca_file
+	end
+
+	if @bearer_token_file.nil? and File.exist?(secret_token_file)
+	  @bearer_token_file = secret_token_file
+	end
+      end
+
+      ssl_options = {
+	client_cert: @client_cert && OpenSSL::X509::Certificate.new(File.read(@client_cert)),
+	client_key:  @client_key && OpenSSL::PKey::RSA.new(File.read(@client_key)),
+	ca_file:     @ca_file,
+	verify_ssl:  @insecure_ssl ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+      }
+
+      auth_options = {}
+      auth_options[:bearer_token] = File.read(@bearer_token_file) if @bearer_token_file
+
+      @client = Kubeclient::Client.new(
+	@kubernetes_url, @api_version,
+	ssl_options: ssl_options,
+	auth_options: auth_options
+      )
+
+      begin
+	@client.api_valid?
+      rescue KubeException => kube_error
+	raise Fluent::ConfigError, "Invalid Kubernetes API #{@api_version} endpoint #{@kubernetes_url}: #{kube_error.message}"
+      end
+    end
+
+    def start_pullers
+      @pull_objects.each(&method(:create_pull_thread))
+    end
+
+    def start_watchers
+      @watchers = @watch_objects.map do |o|
+	o = o.to_h.dup
+	o[:as] = :raw
+	resource_name = o.delete(:resource_name)
+	version = @storage.get(resource_name)
+	o[:resource_version] = version if version
+	@client.public_send("watch_#{resource_name}", o).tap { |watcher|
+	  create_watcher_thread resource_name, watcher
+	}
+      end
+    end
+
+    def create_pull_thread(conf)
+      options = conf.to_h.dup
+      options[:as] = :raw
+      resource_name = options.delete :resource_name
+      pull_interval = options.delete :interval
+
+      thread_create :"pull_#{resource_name}" do
+	tag = generate_tag resource_name
+	while thread_current_running?
+	  log.debug "Going to pull #{resource_name}"
+	  response = @client.public_send "get_#{resource_name}", options
+	  now = Fluent::Engine.now
+	  es = Fluent::MultiEventStream.new
+
+	  # code copied from kubeclient
+	  # kubeclient will create one open struct object for each item in the response,
+	  # but this is totally unecessary in this plugin, thus we use as: :raw.
+	  result = JSON.parse(response)
+
+	  resource_version = result.fetch('resourceVersion') {
+	    result.fetch('metadata', {})['resourceVersion']
+	  }
+
+	  update_op = if resource_version
+			->(item) { item['metadata'].update requestResourceVersion: resource_version }
+		      else
+			->(item) {}
+		      end
+
+	  # result['items'] might be nil due to https://github.com/kubernetes/kubernetes/issues/13096
+	  items = result['items'].to_a
+	  log.debug { "Received #{items.size} #{resource_name}" }
+	  items.each { |item| es.add now, item.tap(&update_op) }
+	  router.emit_stream(tag, es)
+
+	  sleep(pull_interval)
+	end
+      end
+    end
+
+    def create_watcher_thread(object_name, watcher)
+      thread_create(:"watch_#{object_name}") {
+	tag = generate_tag "#{object_name}.watch"
+	watcher.each { |entity|
+	  log.trace { "Received new object from watching #{object_name}"}
+	  entity = JSON.parse(entity)
+	  router.emit tag, Fluent::Engine.now, entity
+	  @storage.put object_name, entity['object']['metadata']['resourceVersion']
+	}
+      }
+    end
+  end
+end

data/test/fluent/plugin/in_kubernetes_objects_test.rb

@@ -0,0 +1,134 @@
+require "test_helper"
+
+describe Fluent::Plugin::KubernetesObjectsInput do
+  include Fluent::Test::Helpers
+  include PluginTestHelper
+
+  before {
+    Fluent::Test.setup # setup router and others
+    stub_k8s_requests
+  }
+    
+  it { expect(::Fluent::Plugin::KubernetesObjectsInput::VERSION).wont_be_nil }
+
+  it "should require at least one <pull> or <watch> section" do
+    expect{create_input_driver("kubernetes_url #{k8s_url}")}.must_raise Fluent::ConfigError
+    expect(create_input_driver(<<~CONF)).wont_be_nil
+    kubernetes_url #{k8s_url}
+    <pull>
+    resource_name nodes
+    </pull>
+    CONF
+    expect(create_input_driver(<<~CONF)).wont_be_nil
+    kubernetes_url #{k8s_url}
+    <watch>
+    resource_name nodes
+    </watch>
+    CONF
+  end
+
+  describe "config: kubernetes_url" do
+    it "should read from environment variables by default" do
+      ENV['KUBERNETES_SERVICE_HOST'] = k8s_host
+      ENV['KUBERNETES_SERVICE_PORT'] = k8s_port
+      expect(create_input_driver(<<~CONF).instance.kubernetes_url).must_equal k8s_url
+      <pull>
+      resource_name nodes
+      </pull>
+      CONF
+    end
+
+    it "should panic if not set" do
+      ENV['KUBERNETES_SERVICE_HOST'] = nil
+      ENV['KUBERNETES_SERVICE_PORT'] = nil
+      expect{ create_input_driver(<<~CONF) }.must_raise Fluent::ConfigError
+      <pull>
+      resource_name nodes
+      </pull>
+      CONF
+    end
+
+    it "should use pick the right path" do
+      ENV['KUBERNETES_SERVICE_HOST'] = k8s_host
+      ENV['KUBERNETES_SERVICE_PORT'] = k8s_port
+      expect(create_input_driver(<<~CONF).instance.kubernetes_url).must_equal k8s_url('apis')
+      api_version apps/v1
+      <pull>
+      resource_name deployments
+      </pull>
+      CONF
+    end
+  end
+
+  describe "emit events" do
+    it "can pull one resource" do
+      d = create_input_driver(<<~CONF)
+      kubernetes_url #{k8s_url}
+      <pull>
+      resource_name nodes
+      </pull>
+      CONF
+
+      d.run expect_emits: 1, timeout: 3
+      events = d.events
+
+      expect(events[0][0]).must_equal 'kubernetes.nodes'
+    end
+
+    it "can pull multiple resources" do
+      d = create_input_driver(<<~CONF)
+      kubernetes_url #{k8s_url}
+      <pull>
+      resource_name namespaces
+      </pull>
+      <pull>
+      resource_name pods
+      </pull>
+      CONF
+
+      d.run expect_emits: 2, timeout: 3
+      events = d.events
+
+      expect(events.any? { |e| e[0] == 'kubernetes.pods'}).must_equal true
+      expect(events.any? { |e| e[0] == 'kubernetes.namespaces'}).must_equal true
+    end
+
+    it "can watch resources" do
+      d = create_input_driver(<<~CONF)
+      kubernetes_url #{k8s_url}
+      <watch>
+      resource_name events
+      </watch>
+      CONF
+
+      d.run expect_emits: 1, timeout: 3
+      events = d.events
+      expect(events.all? { |e| e[0] == 'kubernetes.events.watch'}).must_equal true
+    end
+
+    it "should use checkpoints for watching" do
+      begin
+	require 'tempfile'
+	f = Tempfile.new("fluentd-k8s-objects-test", encoding: 'utf-8')
+	f.write('{"events": "123456"}')
+	f.close
+
+	d = create_input_driver(<<~CONF)
+	kubernetes_url #{k8s_url}
+	<storage>
+	   path #{f.path}
+	</storage>
+	<watch>
+	resource_name events
+	</watch>
+	CONF
+
+	stub_k8s_events params: {resourceVersion: "123456"}
+
+	d.run expect_emits: 1, timeout: 3
+      ensure
+	f.unlink
+      end
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,95 @@
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+#$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
+
+# suppress warning, when require the 'http' library, it shows circle require warnning,
+# which is pretty annoying (kubeclient depends on http for watch_stream)
+_verbose = $VERBOSE
+$VERBOSE = nil
+require "fluent/plugin/in_kubernetes_objects"
+$VERBOSE = _verbose
+
+require "fluent/test"
+require "fluent/test/driver/input"
+require "fluent/test/helpers"
+require "minitest/autorun"
+require "webmock/minitest"
+
+# make assertions from webmock available in minitest/spec
+module Minitest::Expectations
+  infect_an_assertion :assert_requested, :must_be_requested, :reverse
+  infect_an_assertion :assert_not_requested, :wont_be_requested, :reverse
+end
+
+module PluginTestHelper
+  def k8s_host() "127.0.0.1" end
+  def k8s_port() "8001" end
+  def k8s_url(path='api') "https://#{k8s_host}:#{k8s_port}/#{path}" end
+
+  def fluentd_conf_for(*lines)
+    basic_config = [
+    ]
+    (basic_config + lines).join("\n")
+  end
+
+  def create_input_driver(*configs)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::KubernetesObjectsInput).tap { |d|
+      d.configure(fluentd_conf_for(*configs))
+    }
+  end
+
+  def stub_k8s_requests
+    # all stub response bodies are from real k8s 1.8 API calls
+    stub_k8s_api
+    stub_k8s_apis
+    stub_k8s_v1
+    stub_k8s_namespaces
+    stub_k8s_nodes
+    stub_k8s_pods
+    stub_k8s_events
+  end
+
+  def stub_k8s_api
+    open(File.expand_path('../api.json', __FILE__)).tap { |f|
+      stub_request(:get, k8s_url).to_return(body: f.read)
+    }.close
+  end
+
+  def stub_k8s_apis
+    open(File.expand_path('../apis.json', __FILE__)).tap { |f|
+      stub_request(:get, k8s_url('apis')).to_return(body: f.read)
+    }.close
+  end
+
+  def stub_k8s_v1
+    open(File.expand_path('../v1.json', __FILE__)).tap { |f|
+      stub_request(:get, "#{k8s_url}/v1").to_return(body: f.read)
+    }.close
+  end
+
+  def stub_k8s_namespaces
+    open(File.expand_path('../namespaces.json', __FILE__)).tap { |f|
+      stub_request(:get, "#{k8s_url}/v1/namespaces").to_return(body: f.read())
+    }.close
+  end
+
+  def stub_k8s_nodes
+    open(File.expand_path('../nodes.json', __FILE__)).tap { |f|
+      stub_request(:get, "#{k8s_url}/v1/nodes").to_return(body: f.read())
+    }.close
+  end
+
+  def stub_k8s_pods
+    open(File.expand_path('../pods.json', __FILE__)).tap { |f|
+      stub_request(:get, "#{k8s_url}/v1/pods").to_return(body: f.read())
+    }.close
+  end
+
+  def stub_k8s_events(params: nil)
+    open(File.expand_path('../events.json', __FILE__)).tap { |f|
+      url = "#{k8s_url}/v1/watch/events"
+      url << '?' << params.map { |k, v|  "#{k}=#{v}" }.join('&') if params
+      stub_request(:get, url).
+	to_return(body: f.read(), headers: {"Content-Type" => "application/json", "Transfer-Encoding" => "chunked"})
+    }.close
+  end
+end

metadata

@@ -0,0 +1,169 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-kubernetes-objects
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Gimi Liang
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-05-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: kubeclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+description: A Fluentd input plugin for collecting Kubernetes objects, e.g. pods,
+  namespaces, events, etc. by pulling or watching.
+email:
+- zliang@splunk.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-kubernetes-objects.gemspec
+- lib/fluent/plugin/in_kubernetes_objects.rb
+- test/fluent/plugin/in_kubernetes_objects_test.rb
+- test/lib/no_webmock/http_lib_adapters/curb_adapter.rb
+- test/lib/no_webmock/http_lib_adapters/em_http_request_adapter.rb
+- test/lib/no_webmock/http_lib_adapters/excon_adapter.rb
+- test/lib/no_webmock/http_lib_adapters/httpclient_adapter.rb
+- test/lib/no_webmock/http_lib_adapters/manticore_adapter.rb
+- test/lib/no_webmock/http_lib_adapters/patron_adapter.rb
+- test/lib/no_webmock/http_lib_adapters/typhoeus_hydra_adapter.rb
+- test/test_helper.rb
+homepage: https://github.com/splunk/fluent-plugin-kubernetes-objects
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Fluentd Plugin for Kubernetes Objects.
+test_files:
+- test/fluent/plugin/in_kubernetes_objects_test.rb
+- test/lib/no_webmock/http_lib_adapters/patron_adapter.rb
+- test/lib/no_webmock/http_lib_adapters/manticore_adapter.rb
+- test/lib/no_webmock/http_lib_adapters/em_http_request_adapter.rb
+- test/lib/no_webmock/http_lib_adapters/typhoeus_hydra_adapter.rb
+- test/lib/no_webmock/http_lib_adapters/curb_adapter.rb
+- test/lib/no_webmock/http_lib_adapters/httpclient_adapter.rb
+- test/lib/no_webmock/http_lib_adapters/excon_adapter.rb
+- test/test_helper.rb