fluent-plugin-rds-mysql-log 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78512899f0c19ab34bc36d7058c9481cc04dbd5362a82b11a3c02af9e2a541e7
-  data.tar.gz: 414d246d14e327b23737655c77709b1d788297604c41e67b8a4cac5c333b3c7e
+  metadata.gz: a3228cf8c54c9c3bc5a79742976dc646e549cee3a47fdbe2b0b403f3c326badd
+  data.tar.gz: 97e43edd9c455691f4fc7ec881d6d76fdceaf3187587f8df059360c7eef65587
 SHA512:
-  metadata.gz: 6bc9550cc2825150c00536f739552967dba5aca43530c860ea1b4000899ac37b4dd4f3ab1227a88897ba2ec7d74d4e6345ab61d6f71c9724cfa52e1a4dd4fab8
-  data.tar.gz: e998ce6330b33e8ec74e4952032b6e8213e264ccb7fefc6299b8e0c7e3a09a553fe579566e20b92d558bb1ffe6d0ed24c1c9190d51f8a6853bf27962f921a743
+  metadata.gz: 1c96f62c54275181ba8fc95f79f18e8b0106d73f87ef7100d01d95f0abf1d919fd76b79f7be4875056e97bd86460099d0d9d3b334b58448f0a062f0fe278d595
+  data.tar.gz: 179a3823ec2ac89c1b9781a7db5caa99acdf19d1fb9e568b3388ed4e151149238cde851901645003f36be6749e405ca8a7e701c3de0d90c6342016ab2d184292

data/fluent-plugin-rds-mysql-log.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-rds-mysql-log'
-  spec.version       = '0.1.1'
+  spec.version       = '0.1.3'
   spec.authors       = ['Junaid Ali']
   spec.email         = ['jonnie36@yahoo.com']
   spec.summary       = 'Amazon RDS Mysql logs input plugin'
@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log.git'
   spec.license       = 'MIT'
   spec.required_ruby_version = '>= 2.7.0'
-  
+
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})

data/lib/fluent/plugin/in_rds_mysql_log.rb

@@ -75,6 +75,10 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
       log.warn "EC2 Client error occurred: #{e.message}"
     end
   end
+  
+  def is_audit_logs?
+    @pos_info.keys.any? { |log_file_name| log_file_name =~ AUDIT_LOG_PATTERN }
+  end
 
   def get_and_parse_posfile
     begin
@@ -83,21 +87,28 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
 
       pos_last_written_timestamp = 0
       pos_info = {}
+
       File.open(@pos_file, File::RDONLY) do |file|
         file.each_line do |line|
+          
           pos_match = /^(\d+)$/.match(line)
           if pos_match
-            pos_last_written_timestamp = pos_match[1].to_i
+            pos_last_written_timestamp = pos_match[1].to_i 
             log.debug "pos_last_written_timestamp: #{pos_last_written_timestamp}"
           end
 
           pos_match = /^(.+)\t(.+)$/.match(line)
           if pos_match
-            pos_info[pos_match[1]] = pos_match[2]
-            p pos_info
+            if pos_match[1] =~ AUDIT_LOG_PATTERN
+              pos_info[pos_match[1]] = 0
+            else
+              pos_info[pos_match[1]] = pos_match[2]
+            end
+
             log.debug "log_file: #{pos_match[1]}, marker: #{pos_match[2]}"
           end
         end
+
         @pos_last_written_timestamp = pos_last_written_timestamp
         @pos_info = pos_info
       end
@@ -126,11 +137,7 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
     begin
       log.debug "get logfile-list from rds: db_instance_identifier=#{@db_instance_identifier}, pos_last_written_timestamp=#{@pos_last_written_timestamp}"
       
-      # Separate audit logs from other logs
-      
-      audit_logs_exist = @pos_info.keys.any? { |log_file_name| log_file_name =~ AUDIT_LOG_PATTERN }
-      
-      file_last_written = if audit_logs_exist
+      file_last_written = if is_audit_logs?
         # Use custom interval for audit logs
         (Time.now.to_i - @refresh_interval) * 1000
       else
@@ -155,9 +162,12 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
           # save maximum written timestamp value
           @pos_last_written_timestamp = item[:last_written] if @pos_last_written_timestamp < item[:last_written]
 
-          # log file download
-          log_file_name = item[:log_file_name]
-          marker = @pos_info[log_file_name] || "0"
+          log_file_name = item[:log_file_name]          
+          marker = if is_audit_logs?
+            "0"
+          else
+            @pos_info[log_file_name] || "0"
+          end
 
           log.debug "download log from rds: log_file_name=#{log_file_name}, marker=#{marker}"
           logs = @rds.download_db_log_file_portion(

data/test/plugin/test_in_rds_mysql_log.rb

@@ -75,8 +75,6 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
 
     assert_equal(events[0][2]["log_file_name"], 'server_audit.log')
     assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
-
-    # assert_equal(d.instance.instance_variable_get(:@pos_info)['server_audit.log'], 'new_marker')
   end
 
   def test_get_non_audit_log_files
@@ -140,18 +138,10 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
     })
 
     d.instance.instance_variable_set(:@rds, aws_client_stub)
-    
-    d.instance.instance_variable_set(:@pos_last_written_timestamp, (Time.now.to_i - (2*2000)))
-    
-    test_start_time = Time.now.to_i
-    buffer_time = 5 * d.instance.refresh_interval * 20000
-
     d.run(timeout: 3, expect_emits: 1)
     
     events = d.events
     assert_equal(events[0][2]["log_file_name"], 'server_audit.log')
     assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
-
-    assert_operator events[0][1].to_i, :>=, (test_start_time - (buffer_time))
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rds-mysql-log
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Junaid Ali
 bindir: bin
 cert_chain: []
-date: 2025-04-05 00:00:00.000000000 Z
+date: 2025-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-ec2