fluent-plugin-kubernetes_metadata_filter 0.20.0 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5406b418d455a9baa0cd3ae7ea62cc271bb3c501
-  data.tar.gz: ed73b4f32c33630d0ebbe441de8ca39ce85bf591
+  metadata.gz: b20a61e09187754c82416670f602ee2588399078
+  data.tar.gz: ff91664e2431bf6d419b9a9a4ef04367ce0d8dae
 SHA512:
-  metadata.gz: cfc5fdb6c7872041f4663714fae3c22371a613eff290fd1e0bfdc647d6ee4667e43c798f8dfa9e7bd4f1a7bfa37250f8d5a9b00fe3097028ca5e39eb375c858a
-  data.tar.gz: 949e6b18f786f4f3b267b43163834f1a689ed8c2e0226117ed2670db5984ef1a4cfbf2625bffdd77977d5e0096b97771a70e4f539b4c3020158b6941e22cf137
+  metadata.gz: 10d00af65128a6b779fb8aab70a39e0ac20d8cf6b3030514eec564d66e3297e04ea43689cf7296e55ab2174332c0d9aac11425bcc506459798c0820cc45ec4fa
+  data.tar.gz: 2bbd2911b66686211fefcf31f62a076b24b37b3784cf4998f1d5f7e5c9238adc2a1752dd8d9a3645762fbf912f787da3941d84b30d8056a9220ce04d24ed35b6

data/README.md

@@ -24,6 +24,7 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
 * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
 * `merge_json_log` - merge logs in JSON format as top level keys (default: `true`)
+* `preserve_json_log` - preserve JSON logs in raw form in the `log` key, only used if the previous option is true (default: `true`)
 * `de_dot` - replace dots in labels with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
 * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
 

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "0.20.0"
+  gem.version       = "0.21.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -37,6 +37,7 @@ module Fluent
                  :default => 'var\.log\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
     config_param :bearer_token_file, :string, default: nil
     config_param :merge_json_log, :bool, default: true
+    config_param :preserve_json_log, :bool, default: true
     config_param :include_namespace_id, :bool, default: false
     config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
     config_param :de_dot, :bool, default: true
@@ -148,7 +149,7 @@ module Fluent
         begin
           @client.api_valid?
         rescue KubeException => kube_error
-          raise Fluent::ConfigError, "Invalid Kubernetes API endpoint: #{kube_error.message}"
+          raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{kube_error.message}"
         end
 
         if @watch
@@ -224,6 +225,9 @@ module Fluent
         if log[0].eql?('{') && log[-1].eql?('}')
           begin
             record = JSON.parse(log).merge(record)
+            unless @preserve_json_log
+              record.delete('log')
+            end
           rescue JSON::ParserError
           end
         end

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -320,6 +320,18 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_equal(msg.merge(json_log), es.instance_variable_get(:@record_array)[0])
     end
 
+    test 'emit individual fields from json, throw out whole original string' do
+      json_log = {
+        'hello' => 'world',
+        'more' => 'data'
+      }
+      msg = {
+        'log' => "#{json_log.to_json}"
+      }
+      es = emit_with_tag('non-kubernetes', msg, 'preserve_json_log false')
+      assert_equal(json_log, es.instance_variable_get(:@record_array)[0])
+    end
+
     test 'with kubernetes dotted labels, de_dot enabled' do
       VCR.use_cassette('kubernetes_docker_metadata_dotted_labels') do
         es = emit()

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 0.20.0
+  version: 0.21.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-20 00:00:00.000000000 Z
+date: 2016-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd