fluent-plugin-rds-mysql-log 0.1.1 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78512899f0c19ab34bc36d7058c9481cc04dbd5362a82b11a3c02af9e2a541e7
-  data.tar.gz: 414d246d14e327b23737655c77709b1d788297604c41e67b8a4cac5c333b3c7e
+  metadata.gz: 9d9944b03c4974a63ee62fdbb1337b0e230022a300a9f9ddb21c7966ac5917fa
+  data.tar.gz: 9c331cb21f94f2ab6a00cd196dd1a014a449e629be8d8920d7113f5ed41ec930
 SHA512:
-  metadata.gz: 6bc9550cc2825150c00536f739552967dba5aca43530c860ea1b4000899ac37b4dd4f3ab1227a88897ba2ec7d74d4e6345ab61d6f71c9724cfa52e1a4dd4fab8
-  data.tar.gz: e998ce6330b33e8ec74e4952032b6e8213e264ccb7fefc6299b8e0c7e3a09a553fe579566e20b92d558bb1ffe6d0ed24c1c9190d51f8a6853bf27962f921a743
+  metadata.gz: ad22c2ee1f23446bfbe0d19dc9a6a4e4de121d43b7eae3a0c945009019f1adf2d77190a8564039e58f28bd57223be20a44f7f41cbb22ca07371989c9cbccfca4
+  data.tar.gz: f76bf589f490c1e9dd81ba5da40bbf587d53c0a89da1147586f015021fb7dc4a7c00f7aef534451a64bbd07e0976a9150c69fbbef420434f39369428cb67ac02

data/fluent-plugin-rds-mysql-log.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-rds-mysql-log'
-  spec.version       = '0.1.1'
+  spec.version       = '0.1.5'
   spec.authors       = ['Junaid Ali']
   spec.email         = ['jonnie36@yahoo.com']
   spec.summary       = 'Amazon RDS Mysql logs input plugin'
@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log.git'
   spec.license       = 'MIT'
   spec.required_ruby_version = '>= 2.7.0'
-  
+
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})

data/lib/fluent/plugin/in_rds_mysql_log.rb

@@ -75,6 +75,18 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
       log.warn "EC2 Client error occurred: #{e.message}"
     end
   end
+  
+  def is_audit_logs?
+    @pos_info.keys.any? { |log_file_name| log_file_name =~ AUDIT_LOG_PATTERN }
+  end
+  
+  def should_track_marker?(log_file_name)
+    return true if log_file_name == "audit/server_audit.log"
+    return true if log_file_name == "general/mysql-general.log"
+    return true if log_file_name == "error/mysql-error-running.log"
+    
+    false
+  end
 
   def get_and_parse_posfile
     begin
@@ -83,21 +95,23 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
 
       pos_last_written_timestamp = 0
       pos_info = {}
+
       File.open(@pos_file, File::RDONLY) do |file|
         file.each_line do |line|
+          
           pos_match = /^(\d+)$/.match(line)
           if pos_match
-            pos_last_written_timestamp = pos_match[1].to_i
+            pos_last_written_timestamp = pos_match[1].to_i 
             log.debug "pos_last_written_timestamp: #{pos_last_written_timestamp}"
           end
 
           pos_match = /^(.+)\t(.+)$/.match(line)
           if pos_match
             pos_info[pos_match[1]] = pos_match[2]
-            p pos_info
             log.debug "log_file: #{pos_match[1]}, marker: #{pos_match[2]}"
           end
         end
+
         @pos_last_written_timestamp = pos_last_written_timestamp
         @pos_info = pos_info
       end
@@ -106,43 +120,22 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
     end
   end
 
-  def put_posfile
-    # pos file write
-    begin
-      log.debug "pos file write"
-      File.open(@pos_file, File::WRONLY|File::TRUNC) do |file|
-        file.puts @pos_last_written_timestamp.to_s
-
-        @pos_info.each do |log_file_name, marker|
-          file.puts "#{log_file_name}\t#{marker}"
-        end
-      end
-    rescue => e
-      log.warn "pos file write error occurred: #{e.message}"
-    end
-  end
-
   def get_logfile_list
     begin
       log.debug "get logfile-list from rds: db_instance_identifier=#{@db_instance_identifier}, pos_last_written_timestamp=#{@pos_last_written_timestamp}"
       
-      # Separate audit logs from other logs
-      
-      audit_logs_exist = @pos_info.keys.any? { |log_file_name| log_file_name =~ AUDIT_LOG_PATTERN }
-      
-      file_last_written = if audit_logs_exist
-        # Use custom interval for audit logs
-        (Time.now.to_i - @refresh_interval) * 1000
+      if is_audit_logs?
+        @rds.describe_db_log_files(
+          db_instance_identifier: @db_instance_identifier,
+          max_records: 10
+        )
       else
-        # Use default timestamp for other logs
-        @pos_last_written_timestamp
+        @rds.describe_db_log_files(
+          db_instance_identifier: @db_instance_identifier,
+          file_last_written: @pos_last_written_timestamp,
+          max_records: 10,
+        )
       end
-
-      @rds.describe_db_log_files(
-        db_instance_identifier: @db_instance_identifier,
-        file_last_written: file_last_written,
-        max_records: 10,
-      )
     rescue => e
       log.warn "RDS Client describe_db_log_files error occurred: #{e.message}"
     end
@@ -155,9 +148,8 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
           # save maximum written timestamp value
           @pos_last_written_timestamp = item[:last_written] if @pos_last_written_timestamp < item[:last_written]
 
-          # log file download
           log_file_name = item[:log_file_name]
-          marker = @pos_info[log_file_name] || "0"
+          marker = should_track_marker?(log_file_name) ? @pos_info[log_file_name] || "0" : "0"
 
           log.debug "download log from rds: log_file_name=#{log_file_name}, marker=#{marker}"
           logs = @rds.download_db_log_file_portion(
@@ -169,12 +161,7 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
           raw_records = get_logdata(logs)
 
           #emit
-          parse_and_emit(raw_records, log_file_name) unless raw_records.nil?
-          
-          # Save new last_written timestamp **only for non-audit logs**
-          unless log_file_name =~ AUDIT_LOG_PATTERN
-            @pos_last_written_timestamp = item[:last_written] if @pos_last_written_timestamp < item[:last_written]
-          end
+          parse_and_emit(raw_records, log_file_name) unless raw_records&.empty?
         end
       end
     rescue => e
@@ -187,10 +174,12 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
     raw_records = []
     begin
       logs.each do |log|
-        # save got line's marker
-        @pos_info[log_file_name] = log.marker
+        next if log.log_file_data.nil? || log.log_file_data.empty?
 
         raw_records += log.log_file_data.split("\n")
+      
+        # Update marker only if we actually got some data
+        @pos_info[log_file_name] = log.marker if should_track_marker?(log_file_name)
       end
     rescue => e
       log.warn e.message
@@ -252,6 +241,24 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
       log.warn e.message
     end
   end
+  
+  def put_posfile
+    # pos file write
+    begin
+      log.debug "pos file write"
+      File.open(@pos_file, File::WRONLY|File::TRUNC) do |file|
+        log.debug "Writing pos file with timestamp=#{@pos_last_written_timestamp} and pos_info=#{@pos_info.inspect}"
+        @pos_info.delete_if { |file, _| !should_track_marker?(file) }
+        file.puts @pos_last_written_timestamp.to_s
+
+        @pos_info.each do |log_file_name, marker|
+          file.puts "#{log_file_name}\t#{marker}"
+        end
+      end
+    rescue => e
+      log.warn "pos file write error occurred: #{e.message}"
+    end
+  end
 
   class TimerWatcher < Coolio::TimerWatcher
     def initialize(interval, repeat, &callback)

data/test/plugin/test_in_rds_mysql_log.rb

@@ -56,7 +56,7 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
     aws_client_stub = Aws::RDS::Client.new(stub_responses: {
       describe_db_log_files: {
         describe_db_log_files: [
-          { log_file_name: 'server_audit.log', last_written: 123456789, size: 123 }
+          { log_file_name: 'audit/server_audit.log', last_written: 123456789, size: 123 }
         ],
         marker: 'old_marker'
       },
@@ -73,10 +73,8 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
     d.run(timeout: 3, expect_emits: 1)
     events = d.events
 
-    assert_equal(events[0][2]["log_file_name"], 'server_audit.log')
+    assert_equal(events[0][2]["log_file_name"], 'audit/server_audit.log')
     assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
-
-    # assert_equal(d.instance.instance_variable_get(:@pos_info)['server_audit.log'], 'new_marker')
   end
 
   def test_get_non_audit_log_files
@@ -87,7 +85,7 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
       describe_db_log_files: {
         describe_db_log_files: [
           {
-            log_file_name: 'error.log',
+            log_file_name: 'error/mysql-error-running.log',
             last_written: 123456789,
             size: 123
           }
@@ -110,7 +108,7 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
     
     events = d.events
     
-    assert_equal(events[0][2]["log_file_name"], 'error.log')
+    assert_equal(events[0][2]["log_file_name"], 'error/mysql-error-running.log')
     assert_equal(events[0][2]["message"], "IP address '1.2.3.4' could not be resolved: Name or service not known")
 
     # Ensure non-audit logs used `pos_last_written_timestamp`
@@ -120,38 +118,42 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
   def test_get_audit_log_files
     use_iam_role
     d = create_driver
-
+    first_log_line = "20250403 19:41:01,ip-1-1-1-1,service,1.2.3.4,12345678,1234567890,QUERY,test_db,'UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234',0,,"
+    second_log_line = "20250403 19:42:01,ip-1-1-1-1,service,1.2.3.4,12345678,1234567890,QUERY,test_db,'UPDATE table SET id=456, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 5678',0,,"
+    
     aws_client_stub = Aws::RDS::Client.new(stub_responses: {
       describe_db_log_files: {
         describe_db_log_files: [
           {
-            log_file_name: 'server_audit.log',
+            log_file_name: 'audit/server_audit.log',
             last_written: Time.now.to_i,
             size: 123
           }
         ],
         marker: 'marker'
       },
-      download_db_log_file_portion: {
-        log_file_data: "20250403 19:41:01,ip-1-1-1-1,service,1.2.3.4,12345678,1234567890,QUERY,test_db,'UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234',0,,",
-        marker: 'marker',
-        additional_data_pending: false
-      }
+      download_db_log_file_portion: [
+        {
+          log_file_data: first_log_line,
+          marker: 'marker',
+          additional_data_pending: false
+        },
+        {
+          log_file_data: second_log_line,
+          marker: 'marker',
+          additional_data_pending: false
+        }
+      ]
     })
 
     d.instance.instance_variable_set(:@rds, aws_client_stub)
-    
-    d.instance.instance_variable_set(:@pos_last_written_timestamp, (Time.now.to_i - (2*2000)))
-    
-    test_start_time = Time.now.to_i
-    buffer_time = 5 * d.instance.refresh_interval * 20000
-
-    d.run(timeout: 3, expect_emits: 1)
+    d.run(timeout: 3, expect_emits: 2)
     
     events = d.events
-    assert_equal(events[0][2]["log_file_name"], 'server_audit.log')
+    assert_equal(2, events.size)
+    assert_equal(events[0][2]["log_file_name"], 'audit/server_audit.log')
     assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
-
-    assert_operator events[0][1].to_i, :>=, (test_start_time - (buffer_time))
+    assert_equal(events[1][2]["log_file_name"], 'audit/server_audit.log')
+    assert_equal(events[1][2]["message"], "UPDATE table SET id=456, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 5678")
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rds-mysql-log
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.5
 platform: ruby
 authors:
 - Junaid Ali
 bindir: bin
 cert_chain: []
-date: 2025-04-05 00:00:00.000000000 Z
+date: 2025-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-ec2