fluent-plugin-rds-mysql-log 0.1.0 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b755fb7acda5fbc4825668d0351c4ed8b8259127a18027ff372ec1648ea6843
-  data.tar.gz: 4791ffd4877d9eaeba8b28a6674ff1365d096d44b961113c9f1a7ac27aed6115
+  metadata.gz: a3228cf8c54c9c3bc5a79742976dc646e549cee3a47fdbe2b0b403f3c326badd
+  data.tar.gz: 97e43edd9c455691f4fc7ec881d6d76fdceaf3187587f8df059360c7eef65587
 SHA512:
-  metadata.gz: 9450bbed8aa34dfa3a1d4c06606b224d8c0cc11705292c1b6e11d02be963199fe5926c1875a2f34e94e999040c3152b305379a45083ae1fe5418212339c71550
-  data.tar.gz: ba76f33506086245ad6d1c2968466df20b1ca8541a18366ed1170333267bd3b99c860e955a092d547977f2764600ccbc9411d5187d57f85a7e75d0e0d22408d1
+  metadata.gz: 1c96f62c54275181ba8fc95f79f18e8b0106d73f87ef7100d01d95f0abf1d919fd76b79f7be4875056e97bd86460099d0d9d3b334b58448f0a062f0fe278d595
+  data.tar.gz: 179a3823ec2ac89c1b9781a7db5caa99acdf19d1fb9e568b3388ed4e151149238cde851901645003f36be6749e405ca8a7e701c3de0d90c6342016ab2d184292

data/README.md

@@ -7,7 +7,7 @@ This plugin has been created to deal with audit and error logs of mysql.
 
 ## Installation
 
-    $ fluentd-gem install fluent-plugin-rds-mysql-log
+    $ gem install fluent-plugin-rds-mysql-log
 
 ## AWS ELB Settings
 - settings see: [Mysql Database Log Files](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_LogAccess.Concepts.Mysql.html)
@@ -17,7 +17,7 @@ log:
 ```
 SSL_connect returned=1 errno=0 state=SSLv3 read server certificate B: certificate verify failed
 ```
-Do env setting follows:
+Do env settings follows:
 ```
 SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
 ```
@@ -40,7 +40,7 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
 </source>
 ```
 
-### Example setting
+### Example settings
 ```config
 <source>
   type rds_mysql_log

data/fluent-plugin-rds-mysql-log.gemspec

@@ -5,13 +5,14 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-rds-mysql-log'
-  spec.version       = '0.1.0'
+  spec.version       = '0.1.3'
   spec.authors       = ['Junaid Ali']
   spec.email         = ['jonnie36@yahoo.com']
   spec.summary       = 'Amazon RDS Mysql logs input plugin'
   spec.description   = 'fluentd plugin for Amazon RDS Mysql logs input'
   spec.homepage      = 'https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log.git'
   spec.license       = 'MIT'
+  spec.required_ruby_version = '>= 2.7.0'
 
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }

data/lib/fluent/plugin/in_rds_mysql_log.rb

@@ -75,6 +75,10 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
       log.warn "EC2 Client error occurred: #{e.message}"
     end
   end
+  
+  def is_audit_logs?
+    @pos_info.keys.any? { |log_file_name| log_file_name =~ AUDIT_LOG_PATTERN }
+  end
 
   def get_and_parse_posfile
     begin
@@ -83,21 +87,28 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
 
       pos_last_written_timestamp = 0
       pos_info = {}
+
       File.open(@pos_file, File::RDONLY) do |file|
         file.each_line do |line|
+          
           pos_match = /^(\d+)$/.match(line)
           if pos_match
-            pos_last_written_timestamp = pos_match[1].to_i
+            pos_last_written_timestamp = pos_match[1].to_i 
             log.debug "pos_last_written_timestamp: #{pos_last_written_timestamp}"
           end
 
           pos_match = /^(.+)\t(.+)$/.match(line)
           if pos_match
-            pos_info[pos_match[1]] = pos_match[2]
-            p pos_info
+            if pos_match[1] =~ AUDIT_LOG_PATTERN
+              pos_info[pos_match[1]] = 0
+            else
+              pos_info[pos_match[1]] = pos_match[2]
+            end
+
             log.debug "log_file: #{pos_match[1]}, marker: #{pos_match[2]}"
           end
         end
+
         @pos_last_written_timestamp = pos_last_written_timestamp
         @pos_info = pos_info
       end
@@ -126,11 +137,7 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
     begin
       log.debug "get logfile-list from rds: db_instance_identifier=#{@db_instance_identifier}, pos_last_written_timestamp=#{@pos_last_written_timestamp}"
       
-      # Separate audit logs from other logs
-      
-      audit_logs_exist = @pos_info.keys.any? { |log_file_name| log_file_name =~ AUDIT_LOG_PATTERN }
-      
-      file_last_written = if audit_logs_exist
+      file_last_written = if is_audit_logs?
         # Use custom interval for audit logs
         (Time.now.to_i - @refresh_interval) * 1000
       else
@@ -155,9 +162,12 @@ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
           # save maximum written timestamp value
           @pos_last_written_timestamp = item[:last_written] if @pos_last_written_timestamp < item[:last_written]
 
-          # log file download
-          log_file_name = item[:log_file_name]
-          marker = @pos_info[log_file_name] || "0"
+          log_file_name = item[:log_file_name]          
+          marker = if is_audit_logs?
+            "0"
+          else
+            @pos_info[log_file_name] || "0"
+          end
 
           log.debug "download log from rds: log_file_name=#{log_file_name}, marker=#{marker}"
           logs = @rds.download_db_log_file_portion(

data/test/plugin/test_in_rds_mysql_log.rb

@@ -75,8 +75,6 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
 
     assert_equal(events[0][2]["log_file_name"], 'server_audit.log')
     assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
-
-    # assert_equal(d.instance.instance_variable_get(:@pos_info)['server_audit.log'], 'new_marker')
   end
 
   def test_get_non_audit_log_files
@@ -140,18 +138,10 @@ class RdsMysqlLogInputTest < Test::Unit::TestCase
     })
 
     d.instance.instance_variable_set(:@rds, aws_client_stub)
-    
-    d.instance.instance_variable_set(:@pos_last_written_timestamp, (Time.now.to_i - (2*2000)))
-    
-    test_start_time = Time.now.to_i
-    buffer_time = 5 * d.instance.refresh_interval * 20000
-
     d.run(timeout: 3, expect_emits: 1)
     
     events = d.events
     assert_equal(events[0][2]["log_file_name"], 'server_audit.log')
     assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
-
-    assert_operator events[0][1].to_i, :>=, (test_start_time - (buffer_time))
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rds-mysql-log
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.3
 platform: ruby
 authors:
 - Junaid Ali
 bindir: bin
 cert_chain: []
-date: 2025-04-04 00:00:00.000000000 Z
+date: 2025-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-ec2
@@ -155,7 +155,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="