fluent-plugin-kubelet_metadata 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ff16f93534b3c20d748f06d4198c2615c5bcdb2cca97d72d9a32a1b5fedbfc84
+  data.tar.gz: f483fc7148a2a87152d71bbee7f63d958d75da4966693ea4af6a6dc2f31f2de5
+SHA512:
+  metadata.gz: a9596bbf690ce009cff3100874430ed10ff18731c960eb5b0fe0f112233889003e496c9416cbcd60e26b58782830e73811b2bfd5a885373c1eb89b8d44353e4e
+  data.tar.gz: be1be94b1bc528753875028fce7ba73e5c91997c8794c76027a87fbe98e2268dbd55d93c6d11f3ba446580cf74d26dda5001527686d9d60e9a5bc78bfa657d50

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (C) 2013 Michael Grosser <michael@grosser.it>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/fluent/plugin/filter_kubelet_metadata.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+# The file needs to have the name filter_<type> to be auto-discoverable by fluentd
+
+require 'fluent/env'
+require 'fluent/plugin/filter'
+require 'set'
+require 'json'
+require 'uri'
+require 'net/http'
+
+module Fluent::Plugin
+  class KubeletMetadata < Fluent::Plugin::Filter
+    Fluent::Plugin.register_filter('kubelet_metadata', self)
+
+    config_param :statsd, :string, default: nil
+
+    KUBELET_ERROR_BACKOFF_SECONDS = [0.1, 0.5, 1].freeze
+    KUBELET_MAX_REQUESTS_PER_SECOND = 10
+    POD_CACHE_SIZE = 200 # assuming users run 10-100 pods per node
+
+    # rubocop:disable Layout/LineLength
+    # from https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/8f95b0e5fda922ef0576c7ce53d0c72f19a86754/lib/fluent/plugin/filter_kubernetes_metadata.rb#L52
+    # for example: input.kubernetes.pod.var.log.containers.fluentd-mgj9v_default_vault-pki-auth-manager-26f3a7bad715d9d324fb3c818681ec01df14831f0585cb83f2df25a7386ee5f4.log
+    TAG_REGEX = Regexp.compile('var\.log\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$')
+    # rubocop:enable Layout/LineLength
+
+    # - no operations on all values to be fast
+    # - cannot store nil as value
+    class ThreadsafeLruCache
+      def initialize(size)
+        @size = size
+        @data = {}
+        @mutex = Mutex.new
+      end
+
+      def [](key)
+        @mutex.synchronize do
+          value = @data.delete(key) # always remove ... later add it back if necessary
+          return if value.nil? # miss
+          @data[key] = value # mark as recently used
+        end
+      end
+
+      def []=(key, value)
+        @mutex.synchronize do
+          @data.delete @data.first[0] if @data.size == @size # make room
+          @data[key] = value
+        end
+      end
+    end
+
+    def initialize
+      super
+      @cache = ThreadsafeLruCache.new(POD_CACHE_SIZE)
+      @throttle_mutex = Mutex.new
+    end
+
+    def configure(conf)
+      super
+      @statsd = Object.const_get(@statsd) if @statsd
+      fill_cache unless ARGV.include?('--dry-run')
+    end
+
+    def filter(tag, _time, record)
+      return record unless match = tag.match(TAG_REGEX)&.named_captures
+
+      labels = pod_labels(
+        [match.fetch('namespace'), match.fetch('pod_name')],
+        tags: [
+          "pod_name:#{match.fetch('pod_name')}",
+          "namespace:#{match.fetch('namespace')}",
+          "container:#{match.fetch('container_name')}"
+        ]
+      )
+
+      record.merge(
+        'docker' => { 'container_id' => match.fetch('docker_id') },
+        'kubernetes' => {
+          'container_name' => match.fetch('container_name'),
+          'namespace_name' => match.fetch('namespace'),
+          'pod_name' => match.fetch('pod_name'),
+          'labels' => labels
+        }
+      )
+    end
+
+    private
+
+    def pod_labels(key, **args)
+      @cache[key] || begin
+        inc "soft_miss"
+        fill_cache
+        @cache[key] || begin
+          inc "hard_miss", **args # need tags here to be able to debug
+          {}
+        end
+      end
+    end
+
+    # stores each pods labels that kubelet knows in the cache
+    # only storing the labels since pod objects are big
+    def fill_cache
+      pods.each do |pod|
+        @cache[[pod.dig("metadata", "namespace"), pod.dig("metadata", "name")]] = pod.dig("metadata", "labels") || {}
+      end
+    end
+
+    # /runningpods/ has much less data, but does not include initContainerStatuses
+    #
+    # InitContainers are often not available when logs start coming in
+    #
+    # Full kubelet api see https://stackoverflow.com/questions/35075195/is-there-api-documentation-for-kubelet-api
+    def pods
+      retry_on_error backoff: KUBELET_ERROR_BACKOFF_SECONDS do
+        throttle per_second: KUBELET_MAX_REQUESTS_PER_SECOND, throttled: [] do
+          JSON.parse(http_get('https://localhost:10250/pods')).fetch("items")
+        end
+      end
+    rescue StandardError
+      []
+    end
+
+    def http_get(url)
+      uri = URI(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = (uri.scheme == "https")
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      http.open_timeout = 2
+      http.read_timeout = 5
+
+      request = Net::HTTP::Get.new(uri.request_uri)
+      request['Authorization'] = "Bearer #{File.read("/var/run/secrets/kubernetes.io/serviceaccount/token")}"
+
+      response = http.start { http.request request }
+
+      raise "Error response #{response.code} -- #{response.body}" unless response.code == "200"
+
+      response.body
+    end
+
+    def throttle(per_second:, throttled:)
+      t = Process.clock_gettime(Process::CLOCK_MONOTONIC).to_i
+      c = nil
+
+      @throttle_mutex.synchronize do
+        old_t, c = @throttle
+        old_t == t ? c += 1 : c = 1
+        @throttle = [t, c]
+      end
+
+      if c > per_second
+        inc "throttled"
+        throttled
+      else
+        yield
+      end
+    end
+
+    def retry_on_error(backoff:)
+      yield
+    rescue StandardError
+      backoff_index ||= -1
+      backoff_index += 1
+      inc "kubelet_error", tags: ["error:#{$!.class}"]
+      raise unless delay = backoff[backoff_index]
+
+      sleep delay
+      retry
+    end
+
+    def inc(metric, **args)
+      @statsd&.increment "fluentd.kubelet_metadata.#{metric}", **args
+    end
+  end
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-kubelet_metadata
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Michael Grosser
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-10-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: 
+email: michael@grosser.it
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- lib/fluent/plugin/filter_kubelet_metadata.rb
+homepage: https://github.com/grosser/fluent-plugin-kubelet_metadata
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.3
+signing_key: 
+specification_version: 4
+summary: Add metadata to docker logs by asking kubelet api
+test_files: []