fluent-plugin-dockergelf 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 844e49c9984c07d68c6a2295817af4725092cf480399940d2aafbbaea7278cd4
-  data.tar.gz: 766867ba26d2d66200b9b0c8990da50d0d5e66bb5681b65bc3f9791c9af04295
+  metadata.gz: feff450bcb796cff98adfcb03a064c8e95464aaafc1f3719dc2b25c240cfa08a
+  data.tar.gz: 55a274b720520b6ef6fe28b55c227b18cb62398d322c2abf844e52224dfcda21
 SHA512:
-  metadata.gz: 4dd97d20c1d3eea9ba59ad62ec20fa2fe205411811c8b88ec885a4a2d6ac9800d01f5c6bf43865ff3b86837f36629b0e0fa453dab9b472a2750250ee3c1ea6ee
-  data.tar.gz: 8707d5031aae26092b7adc3c84c54eae52cda4603b0e65c6ea3d2184a0fd44aa351e7306f8fe8b39296b3d378a970313e40c35ad446a580239a10e23642ac231
+  metadata.gz: 9b3864c3df3257e36824a128eb8c21ca8885138fb3ba650ee505ff26a8ff10b7ac98f31cdaa2aff47bcc4cd64ca35bc7e7f20d042e91202822028874cf8921be
+  data.tar.gz: 868cefaf9b75722dbacd5e6c3b8cb0b27387cf30ce93ce212ef2c325ab144a6c41dc4ee797795749886ef277c9a6bc995638a9bd3cb30bee8ecf1788f9007390

data/.gitignore

@@ -0,0 +1,2 @@
+pkg/
+.idea/

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-dockergelf (0.1.1)
+    fluent-plugin-dockergelf (0.2.0)
       fluentd (>= 0.14.10, < 2)
 
 GEM

data/README.md

@@ -1,8 +1,19 @@
 # fluent-plugin-dockergelf
 
-[Fluentd](https://fluentd.org/) filter plugin to do something.
+[Fluentd](https://fluentd.org/) filter plugin to handle gelf format log in docker(kubenetes)
 
-TODO: write description for you plugin.
+The log looks like 
+```
+{"log":"{"_instance_id":"","_microservice_id":"","host":"","level":4,"short_message":"Some log information.","timestamp":1539143207.2754333,"version":"1.1"}","stream":"stderr","time":"2018-10-10T05:05:17.102912076Z"}
+```
+This filter will use the timestamp inside as the timestamp. create a new field called logtime as the timestamp from the docker. move the short_message to the log field.
+
+The latest kubernetes_metadata filter does not support the inner json process anymore. So this filter is recommended to put between detect_exceptions and kubernetes_metadata. 
+```clickhouse
+    <filter kubernetes.**>
+      @type dockergelf
+    </filter>
+```
 
 ## Installation
 
@@ -26,12 +37,12 @@ And then execute:
 $ bundle
 ```
 
-## Configuration
+## Release new version
 
-You can generate configuration template:
+You can release new version of this plugin:
 
 ```
-$ fluent-plugin-config-format filter dockergelf
+$ rake release
 ```
 
 You can copy and paste generated documents here.

data/fluent-plugin-dockergelf.gemspec

@@ -3,14 +3,14 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-dockergelf"
-  spec.version = "0.1.5"
+  spec.version = "0.2.0"
   spec.authors = ["Bo Zhang"]
-  spec.email   = ["bo.zhang@inoviagroup.se"]
+  spec.email   = ["bo.b.zhang@ericsson.com"]
 
-  spec.summary       = %q{rrr}
-  spec.description   = %q{ttt}
-  spec.homepage      = "http://google.com"
-  spec.license       = ""
+  spec.summary       = %q{Handles gelf log in docker/kubenetes}
+  spec.description   = %q{Will put docker log time as new field logtime, and use the timestamp in gelf}
+  spec.homepage      = "https://gitlab.apphub.ericsson.net/bo-test-group/fluent-plugin-dockergelf"
+  spec.license       = "Ericsson"
 
   test_files, files  = `git ls-files -z`.split("\x0").partition do |f|
     f.match(%r{^(test|spec|features)/})

data/lib/fluent/plugin/filter_dockergelf.rb

@@ -1,17 +1,39 @@
 require "fluent/plugin/filter"
-
+require "json"
+require "pp"
 module Fluent
   module Plugin
     class DockergelfFilter < Fluent::Plugin::Filter
       Fluent::Plugin.register_filter("dockergelf", self)
+
       def configure(conf)
         super
       end
-      def filter(tag, time, record)
-        record["ttttt"] = tag.to_s
-        record["mmmmm"] = time.to_s
-        record["ddddd"] = record.to_s
-        record
+
+      def filter_with_time(tag, time, record)
+        record["logtime"] = Time.at(time.to_f).iso8601(3).to_s
+        content = record["log"].strip
+        rettime = time
+        if (content =~ /^[{](.*)[}]$/)
+          content_hash = JSON.parse(content)
+          if content_hash["version"].to_s == "1.1" && content_hash["short_message"] != nil #check if gelf format
+            content_hash.delete("version")
+            msg = content_hash["short_message"]
+            content_hash.delete("short_message")
+            content_hash.each do |key ,value|
+              if key == "timestamp"
+                rettime = Fluent::EventTime.from_time(Time.at(value))
+              else
+                record[key]=value
+              end
+            end
+            record["log"] = msg
+          end
+        end
+      rescue
+      ensure
+        puts "returning "+ (Time.at(rettime.to_f).iso8601(3).to_s )+" "+record.to_s
+        return rettime, record
       end
     end
   end

data/test/plugin/test_filter_dockergelf.rb

@@ -16,12 +16,10 @@ class DockergelfFilterTest < Test::Unit::TestCase
     Fluent::Test::Driver::Filter.new(Fluent::Plugin::DockergelfFilter).configure(conf)
   end
 
-  def filter(config, messages)
+  def filterWithTime(config, time, message)
     d = create_driver(config)
-    d.run(default_tag: "input.access") do
-      messages.each do |message|
-        d.feed(message)
-      end
+    d.run() do
+      d.feed("tag.input",time, message)
     end
     d.filtered_records
   end
@@ -44,31 +42,63 @@ class DockergelfFilterTest < Test::Unit::TestCase
   #   # ...
   # end
 
-  sub_test_case 'plugin will add some fields' do
-    test 'add hostname to record' do
+  sub_test_case 'plugin will handle gelf' do
+    test 'parse gel log' do
       conf = CONFIG
-      messages = [
-          {"message" => "This is test message"}
-      ]
-      expected = [
-          {"message" => "This is test message"}
-      ]
-      filtered_records = filter(conf, messages)
-      assert_equal(expected, filtered_records)
+      time = event_time('2018-10-20T00:00:00.123Z')
+      logtime=Time.at(time.to_f).iso8601(3).to_s
+      message = {"log" => '{"_instance_id":"","_microservice_id":"","host":"","level":4,"short_message":"somemsg","timestamp":1539143207.2754333,"version":"1.1"}',
+                  "stream" => 'stderr'
+      }
+      filtered_record = filterWithTime(conf, time, message)[0]
+      puts filtered_record
+      assert_equal(logtime, filtered_record["logtime"])
+      assert_equal(4, filtered_record["level"])
+      assert_equal("stderr", filtered_record["stream"])
+      assert_equal("somemsg", filtered_record["log"])
     end
   end
 
-  sub_test_case 'plugin will add some fields' do
-    test 'add hostname to record' do
+  sub_test_case 'plugin will handle nongelf' do
+    test 'parse nonegelf log' do
       conf = CONFIG
-      messages = [
-          { "message" => "This is test message" }
-      ]
-      expected = [
-          { "message" => "This is test message"}
-      ]
-      filtered_records = filter(conf, messages)
-      assert_equal(expected, filtered_records)
+      time = event_time('2018-10-20T00:00:00.123Z')
+      logtime=Time.at(time.to_f).iso8601(3).to_s
+      message = {"log" => 'abc',
+                 "stream" => 'stderr'
+      }
+      filtered_record = filterWithTime(conf, time, message)[0]
+      puts filtered_record
+      assert_equal(logtime, filtered_record["logtime"])
+      assert_equal("stderr", filtered_record["stream"])
+      assert_equal("abc", filtered_record["log"])
+    end
+    test 'parse bad json log' do
+      conf = CONFIG
+      time = event_time('2018-10-20T00:00:00.123Z')
+      logtime=Time.at(time.to_f).iso8601(3).to_s
+      message = {"log" => '{"asdf":"aa",[}',
+                 "stream" => 'stderr'
+      }
+      filtered_record = filterWithTime(conf, time, message)[0]
+      puts filtered_record
+      assert_equal(logtime, filtered_record["logtime"])
+      assert_equal("stderr", filtered_record["stream"])
+      assert_equal('{"asdf":"aa",[}', filtered_record["log"])
     end
   end
+  #
+  # sub_test_case 'plugin will add some fields' do
+  #   test 'add hostname to record' do
+  #     conf = CONFIG
+  #     messages = [
+  #         { "message" => "This is test message" }
+  #     ]
+  #     expected = [
+  #         { "message" => "This is test message"}
+  #     ]
+  #     filtered_records = filter(conf, messages)
+  #     assert_equal(expected, filtered_records)
+  #   end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-dockergelf
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Bo Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-01 00:00:00.000000000 Z
+date: 2018-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -72,25 +72,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-description: ttt
+description: Will put docker log time as new field logtime, and use the timestamp
+  in gelf
 email:
-- bo.zhang@inoviagroup.se
+- bo.b.zhang@ericsson.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - Gemfile
 - Gemfile.lock
 - README.md
 - Rakefile
 - fluent-plugin-dockergelf.gemspec
 - lib/fluent/plugin/filter_dockergelf.rb
-- pkg/fluent-plugin-dockergelf-0.1.1.gem
 - test/helper.rb
 - test/plugin/test_filter_dockergelf.rb
-homepage: http://google.com
+homepage: https://gitlab.apphub.ericsson.net/bo-test-group/fluent-plugin-dockergelf
 licenses:
-- ''
+- Ericsson
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -111,7 +112,7 @@ rubyforge_project:
 rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
-summary: rrr
+summary: Handles gelf log in docker/kubenetes
 test_files:
 - test/helper.rb
 - test/plugin/test_filter_dockergelf.rb