fluent-plugin-barito 0.3.5 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e4c37eded53d571e098bfbe5b12df7d823711e420229facc6215bfa06f86386
-  data.tar.gz: dc1960dd6c13a8dfb765d8b5ae5275d285e225cc3c02e2ab3ea9e248d805d314
+  metadata.gz: 77b08876975a60787f945406ceef1c8c6885d6c14e9fce2f780e2a11aced749d
+  data.tar.gz: 3422b7d24b4c836dce95fd5987e793ca798fff91691dcf256ff2835253c1f558
 SHA512:
-  metadata.gz: 88db76d4b14e5eab54266cb19a8c217bf9ca704bdca9030aa4f5d151ad2c1ed288875caec7518a197375b2906c201d81b57149f0160c1587fe42de67daf684bf
-  data.tar.gz: 4ceb45b8e712e9d27c929ae2eaa139bc10ef334879aadcaa438ac692b4d157cd7f6c75c659f03468312ea541a80f7451641f58f41cb4cfac40cd5de342358fbc
+  metadata.gz: 9eeb839e61e1a6cfb1eb8223138873e58063e7d546b2106d51dddceb063e1ae8894d8d23782081f3184e4adad8dcd4afc53632fec7d0371651b0c1739e4f42d8
+  data.tar.gz: 85657a49d27eeae2f4928c13c686e586d5349a31e62897a6b8e84687e5b8ef1b374b4c6b7951bfe8da8597f35325919d032188a35b7d85849f0d7d0208ab19a0

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+**0.5.1**
+
+- rename extralabels field to `additional_labels`
+
+**0.5.0**
+
+- Add extralabels field for additional information on client trail. e.g cloud_service_provider or else.
+
+**0.4.0**
+
+- Add custom output that can use dynamic application name format
+
 **0.3.5**
 
 - Add gzip/compressed feature before sent timber
@@ -7,7 +19,7 @@
 **0.3.4**
 
 - Add cluster name field (from k8s)
-  
+
 **0.3.3**
 
 - Print verbose error on logs when receiving failed response from the server

data/fluent-plugin-barito.gemspec

@@ -3,9 +3,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-barito"
-  spec.version = "0.3.5"
+  spec.version = "0.5.1"
   spec.authors = ["BaritoLog"]
-  spec.email   = ["pushm0v.development@gmail.com", "iman.tung@gmail.com"]
+  spec.email   = ["fadli.nurhasan@gmail.com", "gnu.iben@gmail.com"]
 
   spec.summary       = %q{Fluentd output plugin for BaritoLog}
   spec.description   = %q{This gem will forward output from fluentd to Barito-Flow}

data/lib/fluent/plugin/out_barito_batch_k8s.rb

@@ -15,6 +15,7 @@ module Fluent
     config_param :application_name, :string, :default => nil
     config_param :produce_url, :string, :default => ''
     config_param :cluster_name, :string, :default => ''
+    config_param :additional_labels, :hash, :default => {}
 
     # Overide from BufferedOutput
     def start
@@ -51,6 +52,11 @@ module Fluent
           'host' => k8s_metadata['host'],
           'cluster_name' => @cluster_name
         }
+        
+        # Add extra labels from config_params
+        unless @additional_labels.empty?
+          new_timber['client_trail'].merge!(@additional_labels)
+        end
 
         data['items'] << new_timber
       end

data/lib/fluent/plugin/out_barito_batch_vm.rb

@@ -14,6 +14,7 @@ module Fluent
     config_param :application_group_secret, :string, :default => nil
     config_param :application_name, :string, :default => nil
     config_param :produce_url, :string, :default => ''
+    config_param :additional_labels, :hash, :default => {}
 
     # Overide from BufferedOutput
     def start
@@ -35,6 +36,11 @@ module Fluent
         trail = Fluent::Plugin::ClientTrail.new(false)
         timber = Fluent::Plugin::TimberFactory::create_timber(tag, time, record, trail)
 
+        # Add extra labels from config_params
+        unless @additional_labels.empty?
+          timber['client_trail'].merge!(@additional_labels)
+        end
+
         data['items'] << timber
       end
 

data/lib/fluent/plugin/out_barito_dynamic_app_batch_k8s.rb

@@ -0,0 +1,132 @@
+require 'fluent/output'
+require_relative 'barito_timber'
+require_relative 'barito_client_trail'
+require_relative 'barito_transport'
+
+module Fluent
+  class BaritoDynamicAppBatchK8sOutput < BufferedOutput
+
+    PLUGIN_NAME = 'barito_dynamic_app_batch_k8s'
+
+    Fluent::Plugin.register_output(PLUGIN_NAME, self)
+
+    config_param :application_name_format, :string, :default => nil
+    config_param :application_group_secret, :string, :default => nil
+    config_param :produce_url, :string, :default => ''
+    config_param :cluster_name, :string, :default => ''
+    config_param :additional_labels, :hash, :default => {}
+
+    # Overide from BufferedOutput
+    def start
+      super
+    end
+
+    # Overide from BufferedOutput
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    # Overide from BufferedOutput
+    def write(chunk)
+      data = {}
+
+      transport = Fluent::Plugin::BaritoTransport.new(@produce_url, log)
+      chunk.msgpack_each do |tag, time, record|
+
+        # generate application name
+        if @application_name_format.nil?
+          return
+        end
+
+        application_name = expand_application_name_format(record)
+
+        # Kubernetes annotations
+        k8s_metadata = record['kubernetes']
+
+        record = clean_attribute(record)
+        trail = Fluent::Plugin::ClientTrail.new(true)
+        timber = Fluent::Plugin::TimberFactory::create_timber(tag, time, record, trail)
+        new_timber = merge_log_attribute(timber)
+
+        # Add kubernetes information
+        new_timber['k8s_metadata'] = {
+          'pod_name' => k8s_metadata['pod_name'],
+          'namespace_name' => k8s_metadata['namespace_name'],
+          'container_name' => k8s_metadata['container_name'],
+          'host' => k8s_metadata['host'],
+          'cluster_name' => @cluster_name
+        }
+
+        # Add extra labels from config_params
+        unless @additional_labels.empty?
+          new_timber['client_trail'].merge!(@additional_labels)
+        end
+
+        if data[application_name].nil?
+          data[application_name] = { 'items' => [] }
+        end
+        data[application_name]['items'] << new_timber
+      end
+
+      data.each do |application_name, record|
+        header = {
+          content_type: :json,
+          'X-App-Group-Secret' => @application_group_secret,
+          'X-App-Name' => application_name
+        }
+        transport.send_compressed(record, header)
+      end
+    end
+
+    def clean_attribute(record)
+      # Delete kubernetes & docker field
+      record.delete('kubernetes')
+      record.delete('docker')
+      record
+    end
+
+    def merge_log_attribute(record)
+      message_log = nil
+      begin
+        message_log = JSON.parse(record['log'])
+      rescue
+      end
+
+      if !message_log.nil?
+        return record.merge(message_log)
+      end
+
+      record
+    end
+
+    def expand_application_name_format(record)
+      application_name = @application_name_format.dup
+
+      # Regular expression to match placeholders like ${record["key1"]["key2"]}
+      placeholder_regex = /\${record(\["[^"]*"\])+}/
+
+      application_name.gsub!(placeholder_regex) do |placeholder|
+        # Extract keys from the placeholder
+        keys = placeholder.scan(/\["([^"]*)"\]/).flatten
+        # Retrieve the value from the record hash
+        value = get_nested_value(record, keys)
+        value.to_s
+      end
+
+      application_name
+    end
+
+    # Retrieve nested value from record using array of keys
+    def get_nested_value(record, keys)
+      keys.reduce(record) do |acc, key|
+        if acc.is_a?(Hash) && acc.key?(key)
+          acc[key]
+        else
+          # Key not found; return nil to stop further traversal
+          return nil
+        end
+      end
+    end
+
+  end
+end

data/spec/lib/fluent/plugin/out_barito_dynamic_app_batch_k8s_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+
+describe 'Fluent::BaritoBatchDynamicAppK8sOutput' do
+  before do
+    @plugin = Fluent::BaritoDynamicAppBatchK8sOutput.new
+    @plugin.instance_variable_set(:@application_name_format, 'clusterA-${record["kubernetes"]["namespace_name"]}-${record["kubernetes"]["labels"]["app_name"]}')
+  end
+
+  describe '.merge_log_attribute' do
+    it do
+      record = {
+        "kubernetes" => {"some_attr" => "some_value"},
+        "docker" => "docker_value",
+        "log" => "{\"some_attr\": \"info\", \"other_attr\": \"other_value\"}"
+      }
+      new_record = @plugin.merge_log_attribute(record)
+
+      expect(new_record['some_attr']).to eq("info")
+      expect(new_record['other_attr']).to eq("other_value")
+    end
+  end
+
+  describe '.clean_attribute' do
+    it do
+      record = {
+        "kubernetes" => {"some_attr" => "some_value"},
+        "docker" => "docker_value",
+        "attr" => "some_value"
+      }
+      new_record = @plugin.clean_attribute(record)
+
+      expect(new_record['kubernetes']).to be_nil
+      expect(new_record['docker']).to be_nil
+      expect(new_record['attr']).to eq("some_value")
+    end
+  end
+
+  describe '#expand_application_name_format' do
+    it 'expands the application name format with values from the record' do
+      record = {
+        'kubernetes' => {
+          'namespace_name' => 'namespace1',
+          'labels' => {
+            'app_name' => 'app1'
+          }
+        }
+      }
+
+      expanded_name = @plugin.expand_application_name_format(record)
+      expect(expanded_name).to eq('clusterA-namespace1-app1')
+    end
+
+    it 'returns the format string if no placeholders are present' do
+      @plugin.instance_variable_set(:@application_name_format, 'static_name')
+      record = {}
+
+      expanded_name = @plugin.expand_application_name_format(record)
+      expect(expanded_name).to eq('static_name')
+    end
+
+    it 'returns the format string with empty values if keys are missing in the record' do
+      record = {
+        'kubernetes' => {
+          'namespace_name' => 'namespace1'
+        }
+      }
+
+      expanded_name = @plugin.expand_application_name_format(record)
+      expect(expanded_name).to eq('clusterA-namespace1-')
+    end
+
+    it 'returns nil if the format string is nil' do
+      @plugin.instance_variable_set(:@application_name_format, "")
+      record = {}
+
+      expanded_name = @plugin.expand_application_name_format(record)
+      expect(expanded_name).to be_empty
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -115,6 +115,7 @@ RSpec.configure do |config|
   require 'fluent/plugin/out_barito_vm'
   require 'fluent/plugin/out_barito_k8s'
   require 'fluent/plugin/out_barito_batch_k8s'
+  require 'fluent/plugin/out_barito_dynamic_app_batch_k8s'
   require 'fluent/plugin/barito_client_trail'
   require 'fluent/plugin/barito_timber'
   require 'fluent/plugin/barito_transport'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-barito
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.5.1
 platform: ruby
 authors:
 - BaritoLog
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-05-16 00:00:00.000000000 Z
+date: 2024-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -158,8 +158,8 @@ dependencies:
         version: '2'
 description: This gem will forward output from fluentd to Barito-Flow
 email:
-- pushm0v.development@gmail.com
-- iman.tung@gmail.com
+- fadli.nurhasan@gmail.com
+- gnu.iben@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -179,11 +179,13 @@ files:
 - lib/fluent/plugin/barito_transport.rb
 - lib/fluent/plugin/out_barito_batch_k8s.rb
 - lib/fluent/plugin/out_barito_batch_vm.rb
+- lib/fluent/plugin/out_barito_dynamic_app_batch_k8s.rb
 - lib/fluent/plugin/out_barito_k8s.rb
 - lib/fluent/plugin/out_barito_vm.rb
 - spec/lib/fluent/plugin/barito_timber_spec.rb
 - spec/lib/fluent/plugin/barito_transport_spec.rb
 - spec/lib/fluent/plugin/out_barito_batch_k8s_spec.rb
+- spec/lib/fluent/plugin/out_barito_dynamic_app_batch_k8s_spec.rb
 - spec/lib/fluent/plugin/out_barito_k8s_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/BaritoLog/Barito-Fluent-Plugin
@@ -213,5 +215,6 @@ test_files:
 - spec/lib/fluent/plugin/barito_timber_spec.rb
 - spec/lib/fluent/plugin/barito_transport_spec.rb
 - spec/lib/fluent/plugin/out_barito_batch_k8s_spec.rb
+- spec/lib/fluent/plugin/out_barito_dynamic_app_batch_k8s_spec.rb
 - spec/lib/fluent/plugin/out_barito_k8s_spec.rb
 - spec/spec_helper.rb