fluent-plugin-rds-mysql-log 0.1.7 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 27dfa48e97db3adf82530e0e6c13007e1c2ac740edd79405fa00fa8ef2c88754
-  data.tar.gz: b25b6a07e5c24a070f3a659a1e9f1868760256127cde899d6b64902a4ec29ce5
+  metadata.gz: 887e7a7add419cb9fb39dfd6afd859404fb9c58b0cbbfe8eabe89a799ee7e79c
+  data.tar.gz: 4be0d709822ee1ae20c9ea536e39d4ee2ec8d612d10cebf0dee9828975893cd6
 SHA512:
-  metadata.gz: 539e6c23ddd352b9612c28a89351facc816104e94b3e3e61a09df92e50ba31c71034665b3820df97cd9ff80d8a5c3af853c8dfeed1829ef24b2883d73d134e58
-  data.tar.gz: c154ff2e344172096ea11d26355e4adc654a3152b912030f552274152d28d248636fe8597a1dff40ec9a0ccc131def75b2f1dc9bd508366557f19c743f0d17bc
+  metadata.gz: 5e7b69425e6d13d6a4aa7d8cd26338d7fec55e588da45ae2bbf5578ce278e77c215e5af7aaa62c7752c35b14bffd44697921b8e2eba81e72594aa1e5731ff922
+  data.tar.gz: 4e67f019f6e5e3430deff0f079b9b3f2982560eed61dbaaaf97873e6c301a0c89baaa9200ef44faed5168948b3f0f9c4e9adf57563500139532fc6ce7c437097

data/README.md

@@ -1,9 +1,43 @@
 # fluent-plugin-rds-mysql-log
 
 ## Overview
-- Amazon Web Services RDS log input plugin for fluentd
+### AWS RDS log input plugin of Mysql for fluentd
 
-This plugin has been created to deal with audit and error logs of mysql.
+This gem is customized specifically for **MySQL audit logs**, especially tailored for environments where **log rotation** is enabled. In such setups, audit logs rotate every 60 seconds, producing files with a numbered suffix like:
+
+```
+audit/server_audit.log.01 
+audit/server_audit.log.02 
+...
+```
+
+In my use case, I am using the [MariaDB Audit Plugin for MySQL](https://github.com/aws/audit-plugin-for-mysql) to generate these logs.
+
+### Inspiration
+
+This plugin is inspired by [fluent-plugin-rds-pgsql-log](https://github.com/shinsaka/fluent-plugin-rds-pgsql-log). 
+I reused and adapted some of its ideas to handle MySQL audit logs more effectively.
+
+### Handling Markers and Log Rotation
+
+One of the key challenges with audit logs is **marker tracking**, because rotated log file names (like `server_audit.log.01`) are not timestamp-based and can repeat over time. This can cause Fluentd to:
+- Read logs the first time with marker set to `0`
+- Mark the file as "read" using the marker
+- Then skip reading it again, even if new content arrives
+
+To address this, there were two potential approaches:
+
+A general behavior of master log file `audit/server_audit.log` is to receive logs continuously and rotated into `.01`, `.02`, etc., after reaching a size limit. 
+
+#### ✅  Approach 1: Track markers for rotated files (e.g., `server_audit.log.01`, `.02`)
+
+By always reading master file `audit/server_audit.log` from the beginning (`marker = 0`), we avoid duplicates and ensure complete log ingestion. While maintaining marker of rotated files avoid duplication.
+
+#### ❌ Approach 2: Track markers for master file (`audit/server_audit.log`)
+
+By always reading rotated files (e.g., `server_audit.log.01`, `.02`) from the beginning (`marker = 0`), we observed **duplicate log ingestion**.
+
+**Therefore, adopting first approach** — treating the master audit log file as a fresh log source every time, while maintaining markers for rotated files.
 
 ## Installation
 

data/fluent-plugin-rds-mysql-log.gemspec

@@ -5,16 +5,15 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-rds-mysql-log'
-  spec.version       = '0.1.7'
+  spec.version       = '0.1.9'
   spec.authors       = ['Junaid Ali']
   spec.email         = ['jonnie36@yahoo.com']
   spec.summary       = 'Amazon RDS Mysql logs input plugin'
   spec.description   = 'fluentd plugin for Amazon RDS Mysql logs input'
-  spec.homepage      = 'https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log.git'
+  spec.homepage      = 'https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log'
   
-  spec.metadata = {
-    "github_repo" => "ssh://github.com/iamjunaidali/fluent-plugin-rds-mysql-log"
-  }
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = 'https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log'
   
   spec.license       = 'MIT'
   spec.required_ruby_version = '>= 2.7.0'

data/lib/fluent/plugin/in_rds_mysql_log.rb

@@ -221,6 +221,7 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
               "message" => line_match[:query],
               "return_code" => line_match[:retcode],
               "log_file_name" => log_file_name,
+              "detected_level" => "info"
             }
           else
             record = {

data/test/plugin/test_in_rds_mysql_log.rb

@@ -153,7 +153,9 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
     assert_equal(2, events.size)
     assert_equal(events[0][2]["log_file_name"], 'audit/server_audit.log')
     assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
+    assert_equal(events[0][2]["detected_level"], "info")
     assert_equal(events[1][2]["log_file_name"], 'audit/server_audit.log')
     assert_equal(events[1][2]["message"], "UPDATE table SET id=456, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 5678")
+    assert_equal(events[1][2]["detected_level"], "info")
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rds-mysql-log
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.9
 platform: ruby
 authors:
 - Junaid Ali
 bindir: bin
 cert_chain: []
-date: 2025-04-09 00:00:00.000000000 Z
+date: 2025-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-ec2
@@ -144,11 +144,12 @@ files:
 - lib/fluent/plugin/in_rds_mysql_log.rb
 - test/helper.rb
 - test/plugin/test_in_rds_mysql_log.rb
-homepage: https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log.git
+homepage: https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log
 licenses:
 - MIT
 metadata:
-  github_repo: ssh://github.com/iamjunaidali/fluent-plugin-rds-mysql-log
+  homepage_uri: https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log
+  source_code_uri: https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log
 rdoc_options: []
 require_paths:
 - lib