fluent-plugin-rds-mysql-log 0.1.8 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fcba57c8d4b6fc8d25f74197fc6cf800941e55f65c31d1afafa923fdcea142aa
-  data.tar.gz: 7f8ce5fc38eacee479de6cd14052219cda2315263d59fde155252485327a9155
+  metadata.gz: 8c8518eb0a1d3f6518709c2c0c26215dd9ed673a6a58272b434cc0b22e7de513
+  data.tar.gz: 830699a65d0274d7174bc5e5a8fc5cf123c48324cdaefc79ef77966c2996c7ab
 SHA512:
-  metadata.gz: f7cfe921848ccb44ac09611759137f55eb64211c915d75b5257120a3ad7c5f267264c3fd92cf208e238bac0d474872cb57a1c402a7ad9640b8d9643bfb335b06
-  data.tar.gz: 30c7707b42d0cff2d0643ab49c1838b9359a218d34d60e3709f56d5a08e3be1f45bf5635ac3db03d8f2a6d1ce50977b3ec673a028e12ff68251d87ad296ad248
+  metadata.gz: '059a1d93f38aa05262e7c93155ff104d02245b7074780e3455a0effd94557c12913ef9591d9236018501c8355cf0f152bbe533749fddc41373b59584400b38e3'
+  data.tar.gz: a1ab70de3e8746816063793c4ce6934452d6be0e5e278f207b7ee6ed99bfd0b0c896de3cb4ca3711a74174b8d34591d9a92c7e40be6b4dc1070776967af019f0

data/README.md

@@ -1,9 +1,43 @@
 # fluent-plugin-rds-mysql-log
 
 ## Overview
-- Amazon Web Services RDS log input plugin for fluentd
+### AWS RDS log input plugin of Mysql for fluentd
 
-This plugin has been created to deal with audit and error logs of mysql.
+This gem is customized specifically for **MySQL audit logs**, especially tailored for environments where **log rotation** is enabled. In such setups, audit logs rotate every 60 seconds, producing files with a numbered suffix like:
+
+```
+audit/server_audit.log.01 
+audit/server_audit.log.02 
+...
+```
+
+In my use case, I am using the [MariaDB Audit Plugin for MySQL](https://github.com/aws/audit-plugin-for-mysql) to generate these logs.
+
+### Inspiration
+
+This plugin is inspired by [fluent-plugin-rds-pgsql-log](https://github.com/shinsaka/fluent-plugin-rds-pgsql-log). 
+I reused and adapted some of its ideas to handle MySQL audit logs more effectively.
+
+### Handling Markers and Log Rotation
+
+One of the key challenges with audit logs is **marker tracking**, because rotated log file names (like `server_audit.log.01`) are not timestamp-based and can repeat over time. This can cause Fluentd to:
+- Read logs the first time with marker set to `0`
+- Mark the file as "read" using the marker
+- Then skip reading it again, even if new content arrives
+
+To address this, there were two potential approaches:
+
+A general behavior of master log file `audit/server_audit.log` is to receive logs continuously and rotated into `.01`, `.02`, etc., after reaching a size limit. 
+
+#### ✅  Approach 1: Track markers for rotated files (e.g., `server_audit.log.01`, `.02`)
+
+By always reading master file `audit/server_audit.log` from the beginning (`marker = 0`), we avoid duplicates and ensure complete log ingestion. While maintaining marker of rotated files avoid duplication.
+
+#### ❌ Approach 2: Track markers for master file (`audit/server_audit.log`)
+
+By always reading rotated files (e.g., `server_audit.log.01`, `.02`) from the beginning (`marker = 0`), we observed **duplicate log ingestion**.
+
+**Therefore, adopting first approach** — treating the master audit log file as a fresh log source every time, while maintaining markers for rotated files.
 
 ## Installation
 

data/fluent-plugin-rds-mysql-log.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-rds-mysql-log'
-  spec.version       = '0.1.8'
+  spec.version       = '0.1.10'
   spec.authors       = ['Junaid Ali']
   spec.email         = ['jonnie36@yahoo.com']
   spec.summary       = 'Amazon RDS Mysql logs input plugin'

data/lib/fluent/plugin/in_rds_mysql_log.rb

@@ -82,8 +82,6 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
   
   def should_track_marker?(log_file_name)
     return false if log_file_name == "audit/server_audit.log"
-    return false if log_file_name == "general/mysql-general.log"
-    return false if log_file_name == "error/mysql-error-running.log"
     
     true
   end
@@ -221,6 +219,7 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
               "message" => line_match[:query],
               "return_code" => line_match[:retcode],
               "log_file_name" => log_file_name,
+              "detected_level" => "info"
             }
           else
             record = {

data/test/plugin/test_in_rds_mysql_log.rb

@@ -153,7 +153,9 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
     assert_equal(2, events.size)
     assert_equal(events[0][2]["log_file_name"], 'audit/server_audit.log')
     assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
+    assert_equal(events[0][2]["detected_level"], "info")
     assert_equal(events[1][2]["log_file_name"], 'audit/server_audit.log')
     assert_equal(events[1][2]["message"], "UPDATE table SET id=456, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 5678")
+    assert_equal(events[1][2]["detected_level"], "info")
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rds-mysql-log
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.10
 platform: ruby
 authors:
 - Junaid Ali
 bindir: bin
 cert_chain: []
-date: 2025-04-09 00:00:00.000000000 Z
+date: 2025-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-ec2