fluent-plugin-barito 0.3.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d0f25154041dae070b6c1364f5e3aeb3c5e35600bc4c081a40f44d77931ce11c
-  data.tar.gz: 7f9a411aab36e32e4fa0aafeb70b705d9a15b9dc057c5d6af9b141b8e368b85a
+  metadata.gz: 4f97097933e7dbc552c68e40b0234a0fa7f9108bafd9e321bcc26280dcfbcb8f
+  data.tar.gz: af17fc124d2cbbb527c4266124762c14983c96cd9b9dccf4882df0b2af66eca9
 SHA512:
-  metadata.gz: b7084b5fa8bea9b83bd50c92f0ed1d81997b9022e1b668fe9f1ea3744465db95b81f19ecfea3b2a275eb1998754e3b1f7bd594cc60dd821ef0e1b6296500a26f
-  data.tar.gz: eb48ad96d581673acea52f75e1bcf66221918b9841fee8ce062cd5eaa5e0c953ae4bcb386e0c50b71bf3106c83d9f6530c64ba409cde38a82eb8ac1b0fd402a6
+  metadata.gz: a219db3c564d4ebf65aa30b6b03f39c95a3f1ca50280332ee82beaae8c5a7b2068c7f21bc6297dee89bf7869b1cf20ab06476f438526bf143f66063158d251b9
+  data.tar.gz: 26ae3bc887cd80141b64da29a2fabaee88d1da6e2cf3a8b4bafc1d9e272bd7f13c2f94fba128ccc7dfa5cce416edf35b17259d60e2d59b78acc6dda6ceaa7577

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+**0.4.0**
+
+- Add custom output that can use dynamic application name format
+
+**0.3.5**
+
+- Add gzip/compressed feature before sent timber
+
 **0.3.4**
 
 - Add cluster name field (from k8s)

data/fluent-plugin-barito.gemspec

@@ -3,9 +3,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-barito"
-  spec.version = "0.3.4"
+  spec.version = "0.4.0"
   spec.authors = ["BaritoLog"]
-  spec.email   = ["pushm0v.development@gmail.com", "iman.tung@gmail.com"]
+  spec.email   = ["fadli.nurhasan@gmail.com", "gnu.iben@gmail.com"]
 
   spec.summary       = %q{Fluentd output plugin for BaritoLog}
   spec.description   = %q{This gem will forward output from fluentd to Barito-Flow}

data/lib/fluent/plugin/barito_transport.rb

@@ -1,4 +1,5 @@
 require 'rest-client'
+require 'zlib'
 
 class Fluent::Plugin::BaritoTransport
 
@@ -17,4 +18,16 @@ class Fluent::Plugin::BaritoTransport
     end
   end
 
+  def send_compressed(timber, header)
+    begin
+      header['Content-Encoding'] = 'gzip'
+
+      gz = Zlib::GzipWriter.new(StringIO.new)
+      gz << timber.to_json
+      RestClient.post @produce_url, gz.close.string, header
+    rescue Exception => e
+      puts(header)
+      @logger.error [e.message, e.response, Hash[header.collect{|k,v| [k.to_s, v]}]].join(', ')
+    end
+  end
 end

data/lib/fluent/plugin/out_barito_batch_k8s.rb

@@ -66,7 +66,7 @@ module Fluent
         header = {content_type: :json, 'X-App-Secret' => @application_secret}
       end
 
-      transport.send(data, header)
+      transport.send_compressed(data, header)
     end
 
     def clean_attribute(record)

data/lib/fluent/plugin/out_barito_batch_vm.rb

@@ -49,7 +49,7 @@ module Fluent
         header = {content_type: :json, 'X-App-Secret' => @application_secret}
       end
 
-      transport.send(data, header)
+      transport.send_compressed(data, header)
     end
   end
 end

data/lib/fluent/plugin/out_barito_dynamic_app_batch_k8s.rb

@@ -0,0 +1,126 @@
+require 'fluent/output'
+require_relative 'barito_timber'
+require_relative 'barito_client_trail'
+require_relative 'barito_transport'
+
+module Fluent
+  class BaritoDynamicAppBatchK8sOutput < BufferedOutput
+
+    PLUGIN_NAME = 'barito_dynamic_app_batch_k8s'
+
+    Fluent::Plugin.register_output(PLUGIN_NAME, self)
+
+    config_param :application_name_format, :string, :default => nil
+    config_param :application_group_secret, :string, :default => nil
+    config_param :produce_url, :string, :default => ''
+    config_param :cluster_name, :string, :default => ''
+
+    # Overide from BufferedOutput
+    def start
+      super
+    end
+
+    # Overide from BufferedOutput
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    # Overide from BufferedOutput
+    def write(chunk)
+      data = {}
+
+      transport = Fluent::Plugin::BaritoTransport.new(@produce_url, log)
+      chunk.msgpack_each do |tag, time, record|
+
+        # generate application name
+        if @application_name_format.nil?
+          return
+        end
+
+        application_name = expand_application_name_format(record)
+
+        # Kubernetes annotations
+        k8s_metadata = record['kubernetes']
+
+        record = clean_attribute(record)
+        trail = Fluent::Plugin::ClientTrail.new(true)
+        timber = Fluent::Plugin::TimberFactory::create_timber(tag, time, record, trail)
+        new_timber = merge_log_attribute(timber)
+
+        # Add kubernetes information
+        new_timber['k8s_metadata'] = {
+          'pod_name' => k8s_metadata['pod_name'],
+          'namespace_name' => k8s_metadata['namespace_name'],
+          'container_name' => k8s_metadata['container_name'],
+          'host' => k8s_metadata['host'],
+          'cluster_name' => @cluster_name
+        }
+
+        if data[application_name].nil?
+          data[application_name] = { 'items' => [] }
+        end
+        data[application_name]['items'] << new_timber
+      end
+
+      data.each do |application_name, record|
+        header = {
+          content_type: :json,
+          'X-App-Group-Secret' => @application_group_secret,
+          'X-App-Name' => application_name
+        }
+        transport.send_compressed(record, header)
+      end
+    end
+
+    def clean_attribute(record)
+      # Delete kubernetes & docker field
+      record.delete('kubernetes')
+      record.delete('docker')
+      record
+    end
+
+    def merge_log_attribute(record)
+      message_log = nil
+      begin
+        message_log = JSON.parse(record['log'])
+      rescue
+      end
+
+      if !message_log.nil?
+        return record.merge(message_log)
+      end
+
+      record
+    end
+
+    def expand_application_name_format(record)
+      application_name = @application_name_format.dup
+
+      # Regular expression to match placeholders like ${record["key1"]["key2"]}
+      placeholder_regex = /\${record(\["[^"]*"\])+}/
+
+      application_name.gsub!(placeholder_regex) do |placeholder|
+        # Extract keys from the placeholder
+        keys = placeholder.scan(/\["([^"]*)"\]/).flatten
+        # Retrieve the value from the record hash
+        value = get_nested_value(record, keys)
+        value.to_s
+      end
+
+      application_name
+    end
+
+    # Retrieve nested value from record using array of keys
+    def get_nested_value(record, keys)
+      keys.reduce(record) do |acc, key|
+        if acc.is_a?(Hash) && acc.key?(key)
+          acc[key]
+        else
+          # Key not found; return nil to stop further traversal
+          return nil
+        end
+      end
+    end
+
+  end
+end

data/lib/fluent/plugin/out_barito_vm.rb

@@ -42,7 +42,7 @@ module Fluent
         else
           header = {content_type: :json, 'X-App-Secret' => @application_secret}
         end
-        transport.send(timber, header)
+        transport.send_compressed(timber, header)
       end
     end
   end

data/spec/lib/fluent/plugin/out_barito_dynamic_app_batch_k8s_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+
+describe 'Fluent::BaritoBatchDynamicAppK8sOutput' do
+  before do
+    @plugin = Fluent::BaritoDynamicAppBatchK8sOutput.new
+    @plugin.instance_variable_set(:@application_name_format, 'clusterA-${record["kubernetes"]["namespace_name"]}-${record["kubernetes"]["labels"]["app_name"]}')
+  end
+
+  describe '.merge_log_attribute' do
+    it do
+      record = {
+        "kubernetes" => {"some_attr" => "some_value"},
+        "docker" => "docker_value",
+        "log" => "{\"some_attr\": \"info\", \"other_attr\": \"other_value\"}"
+      }
+      new_record = @plugin.merge_log_attribute(record)
+
+      expect(new_record['some_attr']).to eq("info")
+      expect(new_record['other_attr']).to eq("other_value")
+    end
+  end
+
+  describe '.clean_attribute' do
+    it do
+      record = {
+        "kubernetes" => {"some_attr" => "some_value"},
+        "docker" => "docker_value",
+        "attr" => "some_value"
+      }
+      new_record = @plugin.clean_attribute(record)
+
+      expect(new_record['kubernetes']).to be_nil
+      expect(new_record['docker']).to be_nil
+      expect(new_record['attr']).to eq("some_value")
+    end
+  end
+
+  describe '#expand_application_name_format' do
+    it 'expands the application name format with values from the record' do
+      record = {
+        'kubernetes' => {
+          'namespace_name' => 'namespace1',
+          'labels' => {
+            'app_name' => 'app1'
+          }
+        }
+      }
+
+      expanded_name = @plugin.expand_application_name_format(record)
+      expect(expanded_name).to eq('clusterA-namespace1-app1')
+    end
+
+    it 'returns the format string if no placeholders are present' do
+      @plugin.instance_variable_set(:@application_name_format, 'static_name')
+      record = {}
+
+      expanded_name = @plugin.expand_application_name_format(record)
+      expect(expanded_name).to eq('static_name')
+    end
+
+    it 'returns the format string with empty values if keys are missing in the record' do
+      record = {
+        'kubernetes' => {
+          'namespace_name' => 'namespace1'
+        }
+      }
+
+      expanded_name = @plugin.expand_application_name_format(record)
+      expect(expanded_name).to eq('clusterA-namespace1-')
+    end
+
+    it 'returns nil if the format string is nil' do
+      @plugin.instance_variable_set(:@application_name_format, "")
+      record = {}
+
+      expanded_name = @plugin.expand_application_name_format(record)
+      expect(expanded_name).to be_empty
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -115,6 +115,7 @@ RSpec.configure do |config|
   require 'fluent/plugin/out_barito_vm'
   require 'fluent/plugin/out_barito_k8s'
   require 'fluent/plugin/out_barito_batch_k8s'
+  require 'fluent/plugin/out_barito_dynamic_app_batch_k8s'
   require 'fluent/plugin/barito_client_trail'
   require 'fluent/plugin/barito_timber'
   require 'fluent/plugin/barito_transport'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-barito
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors:
 - BaritoLog
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-09 00:00:00.000000000 Z
+date: 2024-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -158,8 +158,8 @@ dependencies:
         version: '2'
 description: This gem will forward output from fluentd to Barito-Flow
 email:
-- pushm0v.development@gmail.com
-- iman.tung@gmail.com
+- fadli.nurhasan@gmail.com
+- gnu.iben@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -179,11 +179,13 @@ files:
 - lib/fluent/plugin/barito_transport.rb
 - lib/fluent/plugin/out_barito_batch_k8s.rb
 - lib/fluent/plugin/out_barito_batch_vm.rb
+- lib/fluent/plugin/out_barito_dynamic_app_batch_k8s.rb
 - lib/fluent/plugin/out_barito_k8s.rb
 - lib/fluent/plugin/out_barito_vm.rb
 - spec/lib/fluent/plugin/barito_timber_spec.rb
 - spec/lib/fluent/plugin/barito_transport_spec.rb
 - spec/lib/fluent/plugin/out_barito_batch_k8s_spec.rb
+- spec/lib/fluent/plugin/out_barito_dynamic_app_batch_k8s_spec.rb
 - spec/lib/fluent/plugin/out_barito_k8s_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/BaritoLog/Barito-Fluent-Plugin
@@ -205,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin for BaritoLog
@@ -213,5 +215,6 @@ test_files:
 - spec/lib/fluent/plugin/barito_timber_spec.rb
 - spec/lib/fluent/plugin/barito_transport_spec.rb
 - spec/lib/fluent/plugin/out_barito_batch_k8s_spec.rb
+- spec/lib/fluent/plugin/out_barito_dynamic_app_batch_k8s_spec.rb
 - spec/lib/fluent/plugin/out_barito_k8s_spec.rb
 - spec/spec_helper.rb