fluent-plugin-uuid2podname 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2760530a2419309be346b3d0531f020e92be9f8a35fafcd082426d5f042d59eb
+  data.tar.gz: 5d3a96095809d20da23eec232e162cb958ae924ee2e4fc6a82b118525cdf7002
+SHA512:
+  metadata.gz: 655ae58ca351ae95c4c93de51d3fcea4b01b11a7934cd955338610235a277a308f7209b6b986a11cea0ea6fb822e833616ea84e999ea832ba7299c76e9333137
+  data.tar.gz: b6e880b4070dd6ddabe17ae77fb780b5394f86dbfaab80c944d45d5458cc27165be440d23dd6fb4798b1ce0e9fa3b5885f41487773886b33c4ddac1bc3fcd52a

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,43 @@
+# fluent-plugin-uuid2podname
+
+[Fluentd](https://fluentd.org/) filter plugin to do something.
+
+TODO: write description for you plugin.
+
+## Installation
+
+### RubyGems
+
+```
+$ gem install fluent-plugin-uuid2podname
+```
+
+### Bundler
+
+Add following line to your Gemfile:
+
+```ruby
+gem "fluent-plugin-uuid2podname"
+```
+
+And then execute:
+
+```
+$ bundle
+```
+
+## Configuration
+
+You can generate configuration template:
+
+```
+$ fluent-plugin-config-format filter uuid2podname
+```
+
+You can copy and paste generated documents here.
+
+## Copyright
+
+* Copyright(c) 2018- SaintLiber
+* License
+  * 

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler"
+Bundler::GemHelper.install_tasks
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push("lib", "test")
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/fluent-plugin-uuid2podname.gemspec

@@ -0,0 +1,27 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name    = "fluent-plugin-uuid2podname"
+  spec.version = "0.0.1"
+  spec.authors = ["geovisor"]
+  spec.email   = ["geovisor@163.com"]
+
+  spec.summary       = "geovis.com.cn"
+  spec.description   = "a fluent plugin by geovisor"
+  spec.homepage      = "http://geovis.com.cn"
+  spec.license       = ""
+
+  test_files, files  = `git ls-files -z`.split("\x0").partition do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.files         = files
+  spec.executables   = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = test_files
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
+end

data/lib/fluent/plugin/filter_uuid2podname.rb

@@ -0,0 +1,195 @@
+
+
+require "fluent/plugin/filter"
+
+module Fluent
+  module Plugin
+    class Uuid2podnameFilter < Fluent::Plugin::Filter
+      Fluent::Plugin.register_filter("uuid2podname", self)
+
+      K8_POD_CA_CERT = 'ca.crt'
+      K8_POD_TOKEN = 'token'
+      desc 'use the k8s insecure port,like 8080,if true then init the http kubeclient'
+      config_param :insecure, :bool, default: false
+      desc 'the kube-apiserver restful url'
+      config_param :kubernetes_url, :string, default: nil
+      desc 'the k8s serviceAccount for comunicating with kube-apiserver'
+      config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
+
+      config_param :apiVersion, :string, default: 'v1'
+      config_param :client_cert, :string, default: nil
+      config_param :client_key, :string, default: nil
+      config_param :ca_file, :string, default: nil
+      config_param :verify_ssl, :bool, default: true
+      config_param :bearer_token_file, :string, default: nil
+
+      def configure(conf)
+        super
+
+        def log.trace?
+          level == Fluent::Log::LEVEL_TRACE
+        end
+        require 'kubeclient'
+
+        log.debug "Uuid2podname Filter configure"
+
+        # Use Kubernetes default service account if we're in a pod.
+        if @kubernetes_url.nil?
+          log.debug "Kubernetes URL is not set - inspecting environment"
+
+          env_host = ENV['KUBERNETES_SERVICE_HOST']
+          env_port = ENV['KUBERNETES_SERVICE_PORT']
+          if !env_host.nil? && !env_port.nil?
+            if insecure
+              @kubernetes_url = "http://#{env_host}:#{env_port}/api"
+            else
+              @kubernetes_url = "https://#{env_host}:#{env_port}/api"
+            end
+            log.debug "Kubernetes URL is now '#{@kubernetes_url}'"
+          end
+        end
+
+        # Use SSL certificate and bearer token from Kubernetes service account.
+        if Dir.exist?(@secret_dir)
+          log.debug "Found directory with secrets: #{@secret_dir}"
+          ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
+          pod_token = File.join(@secret_dir, K8_POD_TOKEN)
+
+          if @ca_file.nil? and File.exist?(ca_cert)
+            log.debug "Found CA certificate: #{ca_cert}"
+            @ca_file = ca_cert
+          end
+
+          if @bearer_token_file.nil? and File.exist?(pod_token)
+            log.debug "Found pod token: #{pod_token}"
+            @bearer_token_file = pod_token
+          end
+        end
+
+        if !@kubernetes_url.nil?
+          if insecure
+            log.debug "Creating insecure K8S client"
+            @client = Kubeclient::Client.new @kubernetes_url, @apiVersion
+          else 
+            ssl_options = {
+                client_cert: !@client_cert.nil? ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
+                client_key:  !@client_key.nil? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
+                ca_file:     @ca_file,
+                verify_ssl:  @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+            }
+
+            auth_options = {}
+
+            if !@bearer_token_file.nil?
+              bearer_token = File.read(@bearer_token_file)
+              auth_options[:bearer_token] = bearer_token
+            end
+
+            log.debug "Creating secure K8S client"
+            @client = Kubeclient::Client.new @kubernetes_url, @apiVersion,
+                                            ssl_options: ssl_options,
+                                            auth_options: auth_options
+          end
+
+          begin
+            @client.api_valid?
+          rescue KubeException => kube_error
+            raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{kube_error.message}"
+          end
+          @podsHash = {}
+          fetch_pods
+        end
+      end
+
+      def filter_stream(tag, es)
+        log.debug "Uuid2podname Filter filter"
+        return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
+        new_es = Fluent::MultiEventStream.new
+        es.each do |time, record|
+          metadata = nil
+          if @podsHash.has_key?(record['uuid'])
+            metadata = @podsHash[record['uuid']]
+            log.debug "find pod for uuid: #{record['uuid']}"
+          else
+            fetch_pods
+            if @podsHash.has_key?(record['uuid'])
+              metadata = @podsHash[record['uuid']]
+            else
+              log.error "pod does not exist: #{record['uuid']}"
+            end 
+          end
+          record = record.merge(metadata) if metadata
+          new_es.add(time, record)
+        end
+
+        new_es
+      end
+
+      def fetch_pods()
+        log.debug "Uuid2podname Filter fetch_pods"
+        newPodsHash = {}
+        @client.get_pods.each do |pod_object|
+          kubernetes_metadata = parse_pod_metadata(pod_object)
+          newPodsHash[kubernetes_metadata['kubernetes']['pod_id']] = kubernetes_metadata
+        end
+        @podsHash = newPodsHash.merge @podsHash
+      end
+
+      def parse_pod_metadata(pod_object)
+        labels = syms_to_strs(pod_object['metadata']['labels'].to_h)
+        if @de_dot
+          self.de_dot!(labels)
+          self.de_dot!(annotations)
+        end
+  
+        # collect container informations
+        container_meta = {}
+        begin
+          pod_object['status']['containerStatuses'].each do|container_status|
+            # get plain container id (eg. docker://hash -> hash)
+            container_id = container_status['containerID'].sub /^[-_a-zA-Z0-9]+:\/\//, ''
+            container_meta[container_id] = {
+                'name' => container_status['name'],
+                'image' => container_status['image'],
+                'image_id' => container_status['imageID']
+            }
+          end
+        rescue
+          log.debug("parsing container meta information failed for: #{pod_object['metadata']['namespace']}/#{pod_object['metadata']['name']} ")
+        end
+  
+        kubernetes_metadata = {
+            'namespace_name' => pod_object['metadata']['namespace'],
+            'pod_id'         => pod_object['metadata']['uid'],
+            'pod_name'       => pod_object['metadata']['name'],
+            'containers'     => syms_to_strs(container_meta),
+            'labels'         => labels,
+            'host'           => pod_object['spec']['nodeName'],
+            'master_url'     => @kubernetes_url
+        }
+        metadata = {
+          'kubernetes' => kubernetes_metadata
+        }
+        return metadata
+      end
+  
+      def syms_to_strs(hsh)
+        newhsh = {}
+        hsh.each_pair do |kk,vv|
+          if vv.is_a?(Hash)
+            vv = syms_to_strs(vv)
+          end
+          if kk.is_a?(Symbol)
+            newhsh[kk.to_s] = vv
+          else
+            newhsh[kk] = vv
+          end
+        end
+        newhsh
+      end
+      
+    end
+  end
+end
+
+

data/test/helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/filter"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_filter_uuid2podname.rb

@@ -0,0 +1,18 @@
+require "helper"
+require "fluent/plugin/filter_uuid2podname.rb"
+
+class Uuid2podnameFilterTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "failure" do
+    flunk
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::Uuid2podnameFilter).configure(conf)
+  end
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-uuid2podname
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- geovisor
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-11-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: a fluent plugin by geovisor
+email:
+- geovisor@163.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-uuid2podname.gemspec
+- lib/fluent/plugin/filter_uuid2podname.rb
+- test/helper.rb
+- test/plugin/test_filter_uuid2podname.rb
+homepage: http://geovis.com.cn
+licenses:
+- ''
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.8
+signing_key: 
+specification_version: 4
+summary: geovis.com.cn
+test_files:
+- test/helper.rb
+- test/plugin/test_filter_uuid2podname.rb