fluent-plugin-barito 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b35472a5c56fd8b341b8fe2c91b1baf8168c7d88078bf5e421a03f14b9c0165
-  data.tar.gz: fddfbfea352eaa235c7599df2ac5920d346f47934aeb3fc14631a509195f3b0a
+  metadata.gz: dddd5d0082d68458cba7621f2f4c01d0ed7f1fed479de55af65acd9b1e5f40be
+  data.tar.gz: e117c347df66acd103f72bb850cb882b5ab35391f1fecf947d68734636f39597
 SHA512:
-  metadata.gz: 409734aba8cc251c54a6cd0e2bb3f2989549f8880d9eb1f58eb2589f79073f989e86bda5f5880dbdd976c4d3fcd4eacd83b2f0c3c13721ca1b66573c88cab01c
-  data.tar.gz: 9d6c49cb7ca58af3a45e0872e7bff01e38fc81002098dc4442ff57a79a1c4833463769554fc5d89769ab398311f5e812e3a9c45153acba470022c650210d267e
+  metadata.gz: ace5ec573a0aca28a58f06ebd1406050a6989d317219441a7cf0a962510e2d745dae06219e469f6adf5ad78a2c3da0ce6b80df44a15c3e89fd53f6e960cd61d5
+  data.tar.gz: 4dd29b131547cd237c28f57a5f4c87a0223cb296e5381965f88dd07b750c83e8eecff3e77337ba8e777649ad822f4db4d4fc7b73d8578d94b2b029be0ec7618a

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+**0.3.2**
+
+- Handle batch produce (from k8s)
+
 **0.3.1**
 
 - Revert nested parsing behaviour

data/fluent-plugin-barito.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-barito"
-  spec.version = "0.3.1"
+  spec.version = "0.3.2"
   spec.authors = ["BaritoLog"]
   spec.email   = ["pushm0v.development@gmail.com", "iman.tung@gmail.com"]
 
@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency 'rspec', '~> 3.4'
   spec.add_development_dependency 'timecop', '~> 0.9.1'
-  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'simplecov', '~> 0'
   spec.add_development_dependency 'coveralls', '~> 0.8.21'
   spec.add_development_dependency 'test-unit', '~> 3.2'
 

data/lib/fluent/plugin/out_barito_batch_k8s.rb

@@ -0,0 +1,89 @@
+require 'fluent/output'
+require 'rest-client'
+require_relative 'barito_timber'
+require_relative 'barito_client_trail'
+
+module Fluent
+  class BaritoBatchK8sOutput < BufferedOutput
+
+    PLUGIN_NAME = 'barito_batch_k8s'
+
+    Fluent::Plugin.register_output(PLUGIN_NAME, self)
+
+    config_param :application_secret, :string, :default => nil
+    config_param :application_group_secret, :string, :default => nil
+    config_param :application_name, :string, :default => nil
+    config_param :produce_url, :string, :default => ''
+
+    # Overide from BufferedOutput
+    def start
+      super
+    end
+
+    # Overide from BufferedOutput
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    # Overide from BufferedOutput
+    def write(chunk)
+      data = {
+        'items' => []
+      }
+      chunk.msgpack_each do |tag, time, record|
+
+        # Kubernetes annotations
+        k8s_metadata = record['kubernetes']
+
+        record = clean_attribute(record)
+        trail = Fluent::Plugin::ClientTrail.new(true)
+        timber = Fluent::Plugin::TimberFactory::create_timber(tag, time, record, trail)
+        new_timber = merge_log_attribute(timber)
+
+        # Add kubernetes information
+        new_timber['k8s_metadata'] = {
+          'pod_name' => k8s_metadata['pod_name'],
+          'namespace_name' => k8s_metadata['namespace_name'],
+          'container_name' => k8s_metadata['container_name'],
+          'host' => k8s_metadata['host']
+        }
+
+        data['items'] << new_timber
+      end
+
+      if @application_secret.nil? or @application_secret.empty?
+          return if @application_group_secret.nil? or @application_name.nil?
+          header = {
+            content_type: :json,
+            'X-App-Group-Secret' => @application_group_secret,
+            'X-App-Name' => @application_name
+          }
+      else
+        header = {content_type: :json, 'X-App-Secret' => @application_secret}
+      end
+
+      response = RestClient.post @produce_url, data.to_json, header
+    end
+
+    def clean_attribute(record)
+      # Delete kubernetes & docker field
+      record.delete('kubernetes')
+      record.delete('docker')
+      record
+    end
+
+    def merge_log_attribute(record)
+      message_log = nil
+      begin
+        message_log = JSON.parse(record['log'])
+      rescue
+      end
+
+      if !message_log.nil?
+        return record.merge(message_log)
+      end
+
+      record
+    end
+  end
+end

data/spec/lib/fluent/plugin/out_barito_batch_k8s_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+describe 'Fluent::BaritoBatchK8sOutput' do
+  describe '.merge_log_attribute' do
+    it do
+      out = Fluent::BaritoBatchK8sOutput.new
+
+      record = {
+        "kubernetes" => {"some_attr" => "some_value"}, 
+        "docker" => "docker_value", 
+        "log" => "{\"some_attr\": \"info\", \"other_attr\": \"other_value\"}"
+      }
+      new_record = out.merge_log_attribute(record)
+
+      expect(new_record['some_attr']).to eq("info")
+      expect(new_record['other_attr']).to eq("other_value")
+    end
+  end
+
+  describe '.clean_attribute' do
+    it do
+      out = Fluent::BaritoBatchK8sOutput.new
+
+      record = {
+        "kubernetes" => {"some_attr" => "some_value"},
+        "docker" => "docker_value",
+        "attr" => "some_value"
+      }
+      new_record = out.clean_attribute(record)
+
+      expect(new_record['kubernetes']).to be_nil
+      expect(new_record['docker']).to be_nil
+      expect(new_record['attr']).to eq("some_value")
+    end
+  end
+
+end

data/spec/lib/fluent/plugin/out_barito_k8s_spec.rb

@@ -55,6 +55,11 @@ describe 'Fluent::BaritoK8sOutput' do
     it do
       out = Fluent::BaritoK8sOutput.new
 
+      record = {
+        "kubernetes" => {"some_attr" => "some_value"}, 
+        "docker" => "docker_value", 
+        "log" => "{\"some_attr\": \"info\", \"other_attr\": \"other_value\"}"
+      }
       new_record = out.merge_log_attribute(record)
 
       expect(new_record['some_attr']).to eq("info")

data/spec/spec_helper.rb

@@ -114,6 +114,7 @@ RSpec.configure do |config|
   require 'fluent/test/driver/output'
   require 'fluent/plugin/out_barito_vm'
   require 'fluent/plugin/out_barito_k8s'
+  require 'fluent/plugin/out_barito_batch_k8s'
   require 'fluent/plugin/barito_client_trail'
   require 'fluent/plugin/barito_timber'
   require 'timecop'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-barito
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - BaritoLog
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-31 00:00:00.000000000 Z
+date: 2018-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -70,14 +70,14 @@ dependencies:
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -162,10 +162,12 @@ files:
 - fluent-plugin-barito.gemspec
 - lib/fluent/plugin/barito_client_trail.rb
 - lib/fluent/plugin/barito_timber.rb
+- lib/fluent/plugin/out_barito_batch_k8s.rb
 - lib/fluent/plugin/out_barito_batch_vm.rb
 - lib/fluent/plugin/out_barito_k8s.rb
 - lib/fluent/plugin/out_barito_vm.rb
 - spec/lib/fluent/plugin/barito_timber_spec.rb
+- spec/lib/fluent/plugin/out_barito_batch_k8s_spec.rb
 - spec/lib/fluent/plugin/out_barito_k8s_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/BaritoLog/Barito-Fluent-Plugin
@@ -194,5 +196,6 @@ specification_version: 4
 summary: Fluentd output plugin for BaritoLog
 test_files:
 - spec/lib/fluent/plugin/barito_timber_spec.rb
+- spec/lib/fluent/plugin/out_barito_batch_k8s_spec.rb
 - spec/lib/fluent/plugin/out_barito_k8s_spec.rb
 - spec/spec_helper.rb