fluent-plugin-rds-mysql-log 0.1.3 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3228cf8c54c9c3bc5a79742976dc646e549cee3a47fdbe2b0b403f3c326badd
-  data.tar.gz: 97e43edd9c455691f4fc7ec881d6d76fdceaf3187587f8df059360c7eef65587
+  metadata.gz: 208c94e798557a8fd899d3c7cc21ab78c0f435896360315c8bda66f765767ced
+  data.tar.gz: 1205db429d34d5b4124680fb7d9dce912f16d86616c57c9aaf670608d03581a3
 SHA512:
-  metadata.gz: 1c96f62c54275181ba8fc95f79f18e8b0106d73f87ef7100d01d95f0abf1d919fd76b79f7be4875056e97bd86460099d0d9d3b334b58448f0a062f0fe278d595
-  data.tar.gz: 179a3823ec2ac89c1b9781a7db5caa99acdf19d1fb9e568b3388ed4e151149238cde851901645003f36be6749e405ca8a7e701c3de0d90c6342016ab2d184292
+  metadata.gz: 593abdfa3a97bec571a4d74de371d42200974423bc27be55ed0e6ef98ba0ec23bdcbdc152d5b600fd632613079925c7d881f1e9c4d0316e3e3d78654c8b97cc6
+  data.tar.gz: 2e94a21e667b1dad11b63f6acebc906081382d3102c5fe0148d753ca84cfee7f2eed23a95ecb7b14f4ebe1e89273b43e11981494d25dc3e599d7abeb2e6aec9e

data/fluent-plugin-rds-mysql-log.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-rds-mysql-log'
-  spec.version       = '0.1.3'
+  spec.version       = '0.1.6'
   spec.authors       = ['Junaid Ali']
   spec.email         = ['jonnie36@yahoo.com']
   spec.summary       = 'Amazon RDS Mysql logs input plugin'

data/lib/fluent/plugin/in_rds_mysql_log.rb

@@ -79,6 +79,14 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
   def is_audit_logs?
     @pos_info.keys.any? { |log_file_name| log_file_name =~ AUDIT_LOG_PATTERN }
   end
+  
+  def should_track_marker?(log_file_name)
+    return false if log_file_name == "audit/server_audit.log"
+    return false if log_file_name == "general/mysql-general.log"
+    return false if log_file_name == "error/mysql-error-running.log"
+    
+    true
+  end
 
   def get_and_parse_posfile
     begin
@@ -99,12 +107,7 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
 
           pos_match = /^(.+)\t(.+)$/.match(line)
           if pos_match
-            if pos_match[1] =~ AUDIT_LOG_PATTERN
-              pos_info[pos_match[1]] = 0
-            else
-              pos_info[pos_match[1]] = pos_match[2]
-            end
-
+            pos_info[pos_match[1]] = pos_match[2]
             log.debug "log_file: #{pos_match[1]}, marker: #{pos_match[2]}"
           end
         end
@@ -117,39 +120,22 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
     end
   end
 
-  def put_posfile
-    # pos file write
-    begin
-      log.debug "pos file write"
-      File.open(@pos_file, File::WRONLY|File::TRUNC) do |file|
-        file.puts @pos_last_written_timestamp.to_s
-
-        @pos_info.each do |log_file_name, marker|
-          file.puts "#{log_file_name}\t#{marker}"
-        end
-      end
-    rescue => e
-      log.warn "pos file write error occurred: #{e.message}"
-    end
-  end
-
   def get_logfile_list
     begin
       log.debug "get logfile-list from rds: db_instance_identifier=#{@db_instance_identifier}, pos_last_written_timestamp=#{@pos_last_written_timestamp}"
       
-      file_last_written = if is_audit_logs?
-        # Use custom interval for audit logs
-        (Time.now.to_i - @refresh_interval) * 1000
+      if is_audit_logs?
+        @rds.describe_db_log_files(
+          db_instance_identifier: @db_instance_identifier,
+          max_records: 10
+        )
       else
-        # Use default timestamp for other logs
-        @pos_last_written_timestamp
+        @rds.describe_db_log_files(
+          db_instance_identifier: @db_instance_identifier,
+          file_last_written: @pos_last_written_timestamp,
+          max_records: 10,
+        )
       end
-
-      @rds.describe_db_log_files(
-        db_instance_identifier: @db_instance_identifier,
-        file_last_written: file_last_written,
-        max_records: 10,
-      )
     rescue => e
       log.warn "RDS Client describe_db_log_files error occurred: #{e.message}"
     end
@@ -162,12 +148,8 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
           # save maximum written timestamp value
           @pos_last_written_timestamp = item[:last_written] if @pos_last_written_timestamp < item[:last_written]
 
-          log_file_name = item[:log_file_name]          
-          marker = if is_audit_logs?
-            "0"
-          else
-            @pos_info[log_file_name] || "0"
-          end
+          log_file_name = item[:log_file_name]
+          marker = should_track_marker?(log_file_name) ? @pos_info[log_file_name] || "0" : "0"
 
           log.debug "download log from rds: log_file_name=#{log_file_name}, marker=#{marker}"
           logs = @rds.download_db_log_file_portion(
@@ -179,12 +161,7 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
           raw_records = get_logdata(logs)
 
           #emit
-          parse_and_emit(raw_records, log_file_name) unless raw_records.nil?
-          
-          # Save new last_written timestamp **only for non-audit logs**
-          unless log_file_name =~ AUDIT_LOG_PATTERN
-            @pos_last_written_timestamp = item[:last_written] if @pos_last_written_timestamp < item[:last_written]
-          end
+          parse_and_emit(raw_records, log_file_name) unless raw_records&.empty?
         end
       end
     rescue => e
@@ -197,10 +174,12 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
     raw_records = []
     begin
       logs.each do |log|
-        # save got line's marker
-        @pos_info[log_file_name] = log.marker
+        next if log.log_file_data.nil? || log.log_file_data.empty?
 
         raw_records += log.log_file_data.split("\n")
+      
+        # Update marker only if we actually got some data
+        @pos_info[log_file_name] = log.marker if should_track_marker?(log_file_name)
       end
     rescue => e
       log.warn e.message
@@ -262,6 +241,24 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
       log.warn e.message
     end
   end
+  
+  def put_posfile
+    # pos file write
+    begin
+      log.debug "pos file write"
+      File.open(@pos_file, File::WRONLY|File::TRUNC) do |file|
+        log.debug "Writing pos file with timestamp=#{@pos_last_written_timestamp} and pos_info=#{@pos_info.inspect}"
+        @pos_info.delete_if { |file, _| !should_track_marker?(file) }
+        file.puts @pos_last_written_timestamp.to_s
+
+        @pos_info.each do |log_file_name, marker|
+          file.puts "#{log_file_name}\t#{marker}"
+        end
+      end
+    rescue => e
+      log.warn "pos file write error occurred: #{e.message}"
+    end
+  end
 
   class TimerWatcher < Coolio::TimerWatcher
     def initialize(interval, repeat, &callback)

data/test/plugin/test_in_rds_mysql_log.rb

@@ -56,7 +56,7 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
     aws_client_stub = Aws::RDS::Client.new(stub_responses: {
       describe_db_log_files: {
         describe_db_log_files: [
-          { log_file_name: 'server_audit.log', last_written: 123456789, size: 123 }
+          { log_file_name: 'audit/server_audit.log', last_written: 123456789, size: 123 }
         ],
         marker: 'old_marker'
       },
@@ -73,7 +73,7 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
     d.run(timeout: 3, expect_emits: 1)
     events = d.events
 
-    assert_equal(events[0][2]["log_file_name"], 'server_audit.log')
+    assert_equal(events[0][2]["log_file_name"], 'audit/server_audit.log')
     assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
   end
 
@@ -85,7 +85,7 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
       describe_db_log_files: {
         describe_db_log_files: [
           {
-            log_file_name: 'error.log',
+            log_file_name: 'error/mysql-error-running.log',
             last_written: 123456789,
             size: 123
           }
@@ -108,7 +108,7 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
     
     events = d.events
     
-    assert_equal(events[0][2]["log_file_name"], 'error.log')
+    assert_equal(events[0][2]["log_file_name"], 'error/mysql-error-running.log')
     assert_equal(events[0][2]["message"], "IP address '1.2.3.4' could not be resolved: Name or service not known")
 
     # Ensure non-audit logs used `pos_last_written_timestamp`
@@ -118,30 +118,42 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
   def test_get_audit_log_files
     use_iam_role
     d = create_driver
-
+    first_log_line = "20250403 19:41:01,ip-1-1-1-1,service,1.2.3.4,12345678,1234567890,QUERY,test_db,'UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234',0,,"
+    second_log_line = "20250403 19:42:01,ip-1-1-1-1,service,1.2.3.4,12345678,1234567890,QUERY,test_db,'UPDATE table SET id=456, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 5678',0,,"
+    
     aws_client_stub = Aws::RDS::Client.new(stub_responses: {
       describe_db_log_files: {
         describe_db_log_files: [
           {
-            log_file_name: 'server_audit.log',
+            log_file_name: 'audit/server_audit.log',
             last_written: Time.now.to_i,
             size: 123
           }
         ],
         marker: 'marker'
       },
-      download_db_log_file_portion: {
-        log_file_data: "20250403 19:41:01,ip-1-1-1-1,service,1.2.3.4,12345678,1234567890,QUERY,test_db,'UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234',0,,",
-        marker: 'marker',
-        additional_data_pending: false
-      }
+      download_db_log_file_portion: [
+        {
+          log_file_data: first_log_line,
+          marker: 'marker',
+          additional_data_pending: false
+        },
+        {
+          log_file_data: second_log_line,
+          marker: 'marker',
+          additional_data_pending: false
+        }
+      ]
     })
 
     d.instance.instance_variable_set(:@rds, aws_client_stub)
-    d.run(timeout: 3, expect_emits: 1)
+    d.run(timeout: 3, expect_emits: 2)
     
     events = d.events
-    assert_equal(events[0][2]["log_file_name"], 'server_audit.log')
+    assert_equal(2, events.size)
+    assert_equal(events[0][2]["log_file_name"], 'audit/server_audit.log')
     assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
+    assert_equal(events[1][2]["log_file_name"], 'audit/server_audit.log')
+    assert_equal(events[1][2]["message"], "UPDATE table SET id=456, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 5678")
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rds-mysql-log
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.6
 platform: ruby
 authors:
 - Junaid Ali
 bindir: bin
 cert_chain: []
-date: 2025-04-07 00:00:00.000000000 Z
+date: 2025-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-ec2