fluent-plugin-rds-mysql-log 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 9b755fb7acda5fbc4825668d0351c4ed8b8259127a18027ff372ec1648ea6843
4
+ data.tar.gz: 4791ffd4877d9eaeba8b28a6674ff1365d096d44b961113c9f1a7ac27aed6115
5
+ SHA512:
6
+ metadata.gz: 9450bbed8aa34dfa3a1d4c06606b224d8c0cc11705292c1b6e11d02be963199fe5926c1875a2f34e94e999040c3152b305379a45083ae1fe5418212339c71550
7
+ data.tar.gz: ba76f33506086245ad6d1c2968466df20b1ca8541a18366ed1170333267bd3b99c860e955a092d547977f2764600ccbc9411d5187d57f85a7e75d0e0d22408d1
data/.gitignore ADDED
@@ -0,0 +1,5 @@
1
+ *.gem
2
+ fluent.conf
3
+ coverage/
4
+ mysql-log-pos.dat
5
+ Gemfile.lock
data/Gemfile ADDED
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ source 'https://rubygems.org'
4
+
5
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Junaid Ali
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,90 @@
1
+ # fluent-plugin-rds-mysql-log
2
+
3
+ ## Overview
4
+ - Amazon Web Services RDS log input plugin for fluentd
5
+
6
+ This plugin has been created to deal with audit and error logs of mysql.
7
+
8
+ ## Installation
9
+
10
+ $ fluentd-gem install fluent-plugin-rds-mysql-log
11
+
12
+ ## AWS ELB Settings
13
+ - settings see: [Mysql Database Log Files](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_LogAccess.Concepts.Mysql.html)
14
+
15
+ ## When SSL certification error
16
+ log:
17
+ ```
18
+ SSL_connect returned=1 errno=0 state=SSLv3 read server certificate B: certificate verify failed
19
+ ```
20
+ Do env setting follows:
21
+ ```
22
+ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
23
+ ```
24
+
25
+ ## Configuration
26
+
27
+ ```config
28
+ <source>
29
+ type rds_mysql_log
30
+ # required
31
+ region <region name>
32
+ db_instance_identifier <instance identifier>
33
+ # optional if you can IAM credentials
34
+ access_key_id <access_key>
35
+ secret_access_key <secret_access_key>
36
+ # optional
37
+ refresh_interval <interval number by second(default: 30)>
38
+ tag <tag name(default: rds-mysql.log>
39
+ pos_file <log getting position file(default: rds-mysql.log)>
40
+ </source>
41
+ ```
42
+
43
+ ### Example setting
44
+ ```config
45
+ <source>
46
+ type rds_mysql_log
47
+ region eu-central-1
48
+ db_instance_identifier test-mysql
49
+ access_key_id XXXXXXXXXXXXXXXXXXXX
50
+ secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
51
+ refresh_interval 30
52
+ tag mysql.log
53
+ pos_file /tmp/mysql-log-pos.dat
54
+ </source>
55
+
56
+ <match mysql.log>
57
+ type stdout
58
+ </match>
59
+ ```
60
+
61
+ ### json output example
62
+ ```
63
+ Audit Logs:
64
+
65
+ {
66
+ "time" => "20250403 19:41:01",
67
+ "serverhost" => "ip-1-1-1-1",
68
+ "host" => "1.2.3.4",
69
+ "user" => "service",
70
+ "database" => "test_db",
71
+ "queryid" => "1234567890",
72
+ "connectionid" => "12345678",
73
+ "message" => "UPDATE table SET id=123, updated_at='2025-04-03 19:38:08.681797', is_weight_saved=1 WHERE table.id = 1234",
74
+ "return_code" => "0",
75
+ "log_file_name" => "server_audit.log"
76
+ }
77
+
78
+ Error Logs:
79
+
80
+ {
81
+ "time" => "2025-03-21T00:00:04.275032Z",
82
+ "thread_id" => "123456789",
83
+ "severity" => "Warning",
84
+ "error_code" => "MY-010055",
85
+ "subsystem" => "Server",
86
+ "message" => "IP address '1.2.3.4' could not be resolved: Name or service not known",
87
+ "log_file_name" => "error.log"
88
+ }
89
+
90
+ ```
data/Rakefile ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env rake
2
+ # frozen_string_literal: true
3
+
4
+ ENV['TZ'] = 'UTC'
5
+ require 'bundler/gem_tasks'
6
+
7
+ require 'rake/testtask'
8
+ Rake::TestTask.new(:test) do |test|
9
+ test.libs << 'lib' << 'test'
10
+ test.pattern = 'test/**/test_*.rb'
11
+ test.verbose = true
12
+ end
13
+
14
+ task default: :test
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ lib = File.expand_path('lib', __dir__)
4
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'fluent-plugin-rds-mysql-log'
8
+ spec.version = '0.1.0'
9
+ spec.authors = ['Junaid Ali']
10
+ spec.email = ['jonnie36@yahoo.com']
11
+ spec.summary = 'Amazon RDS Mysql logs input plugin'
12
+ spec.description = 'fluentd plugin for Amazon RDS Mysql logs input'
13
+ spec.homepage = 'https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log.git'
14
+ spec.license = 'MIT'
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ['lib']
20
+
21
+ spec.add_dependency 'aws-sdk-ec2', '~> 1.5'
22
+ spec.add_dependency 'aws-sdk-rds', '~> 1.2'
23
+ spec.add_dependency 'fluentd', '>= 0.14.0', '< 2'
24
+
25
+ spec.add_development_dependency 'bundler', '~> 2.6'
26
+ spec.add_development_dependency 'rake', '~> 13.2'
27
+ spec.add_development_dependency 'simplecov', '~>0.22'
28
+ spec.add_development_dependency 'test-unit', '~> 3.6'
29
+ spec.add_development_dependency 'webmock', '~>3.2'
30
+ end
@@ -0,0 +1,14 @@
1
+ <source>
2
+ @type rds_mysql_log
3
+ access_key_id XXXXXXXXXXXXXXXXXXXX
4
+ secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
5
+ region eu-central-1
6
+ db_instance_identifier test-mysql
7
+ refresh_interval 30
8
+ tag mysql.log
9
+ pos_file /tmp/mysql-log-pos.dat
10
+ </source>
11
+
12
+ <match mysql.log>
13
+ @type stdout
14
+ </match>
@@ -0,0 +1,267 @@
1
+ require 'fluent/input'
2
+ require 'aws-sdk-ec2'
3
+ require 'aws-sdk-rds'
4
+
5
+ class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
6
+ Fluent::Plugin.register_input('rds_mysql_log', self)
7
+
8
+ helpers :timer
9
+
10
+ LOG_REGEXP = /^(?:(?<audit_logs>(?<timestamp>(\d{8})\s(\d{2}:\d{2}:\d{2})?),(?<serverhost>[^,]+?),(?<username>[^,]+?),(?<host>[^,]+?),(?<connectionid>[^,]+?),(?<queryid>[^,]+?),(?<operation>[^,]+?),(?<database>[^,]+?),'(?<query>.*?)',(?<retcode>[0-9]?),(?:,)?)|(?<other_logs>(?<time>(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2}\.\d+Z)?)\s(?<thread_id>\d+?)\s\[(?<severity>([^\]]+)?)\]\s\[(?<error_code>([^\]]+)?)\]\s\[(?<subsystem>([^\]]+)?)\]\s((?<message>.+)?)))$/m
11
+ AUDIT_LOG_PATTERN = /server_audit\.log(\.\d+)?$/i
12
+
13
+ config_param :access_key_id, :string, :default => nil
14
+ config_param :secret_access_key, :string, :default => nil
15
+ config_param :region, :string, :default => nil
16
+ config_param :db_instance_identifier, :string, :default => nil
17
+ config_param :pos_file, :string, :default => "fluent-plugin-rds-mysql-log-pos.dat"
18
+ config_param :refresh_interval, :integer, :default => 30
19
+ config_param :tag, :string, :default => "rds-mysql.log"
20
+
21
+ def configure(conf)
22
+ super
23
+
24
+ raise Fluent::ConfigError.new("region is required") unless @region
25
+ if !has_iam_role?
26
+ raise Fluent::ConfigError.new("access_key_id is required") if @access_key_id.nil?
27
+ raise Fluent::ConfigError.new("secret_access_key is required") if @secret_access_key.nil?
28
+ end
29
+ raise Fluent::ConfigError.new("db_instance_identifier is required") unless @db_instance_identifier
30
+ raise Fluent::ConfigError.new("pos_file is required") unless @pos_file
31
+ raise Fluent::ConfigError.new("refresh_interval is required") unless @refresh_interval
32
+ raise Fluent::ConfigError.new("tag is required") unless @tag
33
+
34
+ begin
35
+ options = {
36
+ :region => @region,
37
+ }
38
+ if @access_key_id && @secret_access_key
39
+ options[:access_key_id] = @access_key_id
40
+ options[:secret_access_key] = @secret_access_key
41
+ end
42
+ @rds = Aws::RDS::Client.new(options)
43
+ rescue => e
44
+ log.warn "RDS Client error occurred: #{e.message}"
45
+ end
46
+ end
47
+
48
+ def start
49
+ super
50
+
51
+ # pos file touch
52
+ File.open(@pos_file, File::RDWR|File::CREAT).close
53
+
54
+ timer_execute(:poll_logs, @refresh_interval, repeat: true, &method(:input))
55
+ end
56
+
57
+ def shutdown
58
+ super
59
+ end
60
+
61
+ private
62
+
63
+ def input
64
+ get_and_parse_posfile
65
+ log_files = get_logfile_list
66
+ get_logfile(log_files)
67
+ put_posfile
68
+ end
69
+
70
+ def has_iam_role?
71
+ begin
72
+ ec2 = Aws::EC2::Client.new(region: @region)
73
+ !ec2.config.credentials.nil?
74
+ rescue => e
75
+ log.warn "EC2 Client error occurred: #{e.message}"
76
+ end
77
+ end
78
+
79
+ def get_and_parse_posfile
80
+ begin
81
+ # get & parse pos file
82
+ log.debug "pos file get start"
83
+
84
+ pos_last_written_timestamp = 0
85
+ pos_info = {}
86
+ File.open(@pos_file, File::RDONLY) do |file|
87
+ file.each_line do |line|
88
+ pos_match = /^(\d+)$/.match(line)
89
+ if pos_match
90
+ pos_last_written_timestamp = pos_match[1].to_i
91
+ log.debug "pos_last_written_timestamp: #{pos_last_written_timestamp}"
92
+ end
93
+
94
+ pos_match = /^(.+)\t(.+)$/.match(line)
95
+ if pos_match
96
+ pos_info[pos_match[1]] = pos_match[2]
97
+ p pos_info
98
+ log.debug "log_file: #{pos_match[1]}, marker: #{pos_match[2]}"
99
+ end
100
+ end
101
+ @pos_last_written_timestamp = pos_last_written_timestamp
102
+ @pos_info = pos_info
103
+ end
104
+ rescue => e
105
+ log.warn "pos file get and parse error occurred: #{e.message}"
106
+ end
107
+ end
108
+
109
+ def put_posfile
110
+ # pos file write
111
+ begin
112
+ log.debug "pos file write"
113
+ File.open(@pos_file, File::WRONLY|File::TRUNC) do |file|
114
+ file.puts @pos_last_written_timestamp.to_s
115
+
116
+ @pos_info.each do |log_file_name, marker|
117
+ file.puts "#{log_file_name}\t#{marker}"
118
+ end
119
+ end
120
+ rescue => e
121
+ log.warn "pos file write error occurred: #{e.message}"
122
+ end
123
+ end
124
+
125
+ def get_logfile_list
126
+ begin
127
+ log.debug "get logfile-list from rds: db_instance_identifier=#{@db_instance_identifier}, pos_last_written_timestamp=#{@pos_last_written_timestamp}"
128
+
129
+ # Separate audit logs from other logs
130
+
131
+ audit_logs_exist = @pos_info.keys.any? { |log_file_name| log_file_name =~ AUDIT_LOG_PATTERN }
132
+
133
+ file_last_written = if audit_logs_exist
134
+ # Use custom interval for audit logs
135
+ (Time.now.to_i - @refresh_interval) * 1000
136
+ else
137
+ # Use default timestamp for other logs
138
+ @pos_last_written_timestamp
139
+ end
140
+
141
+ @rds.describe_db_log_files(
142
+ db_instance_identifier: @db_instance_identifier,
143
+ file_last_written: file_last_written,
144
+ max_records: 10,
145
+ )
146
+ rescue => e
147
+ log.warn "RDS Client describe_db_log_files error occurred: #{e.message}"
148
+ end
149
+ end
150
+
151
+ def get_logfile(log_files)
152
+ begin
153
+ log_files.each do |log_file|
154
+ log_file.describe_db_log_files.each do |item|
155
+ # save maximum written timestamp value
156
+ @pos_last_written_timestamp = item[:last_written] if @pos_last_written_timestamp < item[:last_written]
157
+
158
+ # log file download
159
+ log_file_name = item[:log_file_name]
160
+ marker = @pos_info[log_file_name] || "0"
161
+
162
+ log.debug "download log from rds: log_file_name=#{log_file_name}, marker=#{marker}"
163
+ logs = @rds.download_db_log_file_portion(
164
+ db_instance_identifier: @db_instance_identifier,
165
+ log_file_name: log_file_name,
166
+ marker: marker,
167
+ )
168
+
169
+ raw_records = get_logdata(logs)
170
+
171
+ #emit
172
+ parse_and_emit(raw_records, log_file_name) unless raw_records.nil?
173
+
174
+ # Save new last_written timestamp **only for non-audit logs**
175
+ unless log_file_name =~ AUDIT_LOG_PATTERN
176
+ @pos_last_written_timestamp = item[:last_written] if @pos_last_written_timestamp < item[:last_written]
177
+ end
178
+ end
179
+ end
180
+ rescue => e
181
+ log.warn e.message
182
+ end
183
+ end
184
+
185
+ def get_logdata(logs)
186
+ log_file_name = logs.context.params[:log_file_name]
187
+ raw_records = []
188
+ begin
189
+ logs.each do |log|
190
+ # save got line's marker
191
+ @pos_info[log_file_name] = log.marker
192
+
193
+ raw_records += log.log_file_data.split("\n")
194
+ end
195
+ rescue => e
196
+ log.warn e.message
197
+ end
198
+ return raw_records
199
+ end
200
+
201
+ def event_time_of_row(record)
202
+ time = Time.parse(record["time"])
203
+ return Fluent::EventTime.from_time(time)
204
+ end
205
+
206
+ def parse_and_emit(raw_records, log_file_name)
207
+ begin
208
+ log.debug "raw_records.count: #{raw_records.count}"
209
+
210
+ record = nil
211
+ raw_records.each do |raw_record|
212
+ log.debug "raw_record=#{raw_record}"
213
+ line_match = LOG_REGEXP.match(raw_record)
214
+
215
+ unless line_match
216
+ # combine chain of log
217
+ record["message"] << "\n" + raw_record unless record.nil?
218
+ else
219
+ # emit before record
220
+ router.emit(@tag, event_time_of_row(record), record) unless record.nil?
221
+
222
+ # set a record
223
+ if line_match[:audit_logs]
224
+ record = {
225
+ "time" => line_match[:timestamp],
226
+ "serverhost" => line_match[:serverhost],
227
+ "host" => line_match[:host],
228
+ "user" => line_match[:username],
229
+ "database" => line_match[:database],
230
+ "queryid" => line_match[:queryid],
231
+ "connectionid" => line_match[:connectionid],
232
+ "message" => line_match[:query],
233
+ "return_code" => line_match[:retcode],
234
+ "log_file_name" => log_file_name,
235
+ }
236
+ else
237
+ record = {
238
+ "time" => line_match[:time],
239
+ "thread_id" => line_match[:thread_id],
240
+ "severity" => line_match[:severity],
241
+ "error_code" => line_match[:error_code],
242
+ "subsystem" => line_match[:subsystem],
243
+ "message" => line_match[:message],
244
+ "log_file_name" => log_file_name,
245
+ }
246
+ end
247
+ end
248
+ end
249
+ # emit last record
250
+ router.emit(@tag, event_time_of_row(record), record) unless record.nil?
251
+ rescue => e
252
+ log.warn e.message
253
+ end
254
+ end
255
+
256
+ class TimerWatcher < Coolio::TimerWatcher
257
+ def initialize(interval, repeat, &callback)
258
+ @callback = callback
259
+ on_timer # first call
260
+ super(interval, repeat)
261
+ end
262
+
263
+ def on_timer
264
+ @callback.call
265
+ end
266
+ end
267
+ end
data/test/helper.rb ADDED
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/setup'
4
+ require 'test/unit'
5
+ require 'webmock/test_unit'
6
+ require 'simplecov'
7
+
8
+ SimpleCov.start do
9
+ add_filter '/test/'
10
+ end
11
+
12
+ $LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
13
+ $LOAD_PATH.unshift(__dir__)
14
+ require 'fluent/test'
15
+ require 'fluent/test/helpers'
16
+ require 'fluent/test/driver/input'
17
+ require 'fluent/plugin/in_rds_mysql_log'
18
+
19
+ module Test
20
+ module Unit
21
+ class TestCase
22
+ include Fluent::Test::Helpers
23
+ extend Fluent::Test::Helpers
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,157 @@
1
+ require_relative '../helper'
2
+
3
+ class RdsMysqlLogInputTest < Test::Unit::TestCase
4
+ def setup
5
+ Fluent::Test.setup
6
+ end
7
+
8
+ DEFAULT_CONFIG = {
9
+ access_key_id: 'dummy_access_key_id',
10
+ secret_access_key: 'dummy_secret_access_key',
11
+ region: 'ap-northeast-1',
12
+ db_instance_identifier: 'test-mysql-id',
13
+ refresh_interval: 2,
14
+ pos_file: 'mysql-log-pos.dat',
15
+ }
16
+
17
+ def parse_config(conf = {})
18
+ ''.tap{|s| conf.each { |k, v| s << "#{k} #{v}\n" } }
19
+ end
20
+
21
+ def create_driver(conf = DEFAULT_CONFIG)
22
+ Fluent::Test::Driver::Input.new(Fluent::Plugin::RdsMysqlLogInput).configure(parse_config conf)
23
+ end
24
+
25
+ def iam_info_url
26
+ 'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
27
+ end
28
+
29
+ def use_iam_role
30
+ stub_request(:get, iam_info_url)
31
+ .to_return(status: [200, 'OK'], body: "hostname")
32
+ stub_request(:get, "#{iam_info_url}hostname")
33
+ .to_return(status: [200, 'OK'],
34
+ body: {
35
+ "AccessKeyId" => "dummy",
36
+ "SecretAccessKey" => "secret",
37
+ "Token" => "token"
38
+ }.to_json)
39
+ end
40
+
41
+ def test_configure
42
+ use_iam_role
43
+ d = create_driver
44
+ assert_equal 'dummy_access_key_id', d.instance.access_key_id
45
+ assert_equal 'dummy_secret_access_key', d.instance.secret_access_key
46
+ assert_equal 'ap-northeast-1', d.instance.region
47
+ assert_equal 'test-mysql-id', d.instance.db_instance_identifier
48
+ assert_equal 'mysql-log-pos.dat', d.instance.pos_file
49
+ assert_equal 2, d.instance.refresh_interval
50
+ end
51
+
52
+ def test_audit_log_marker_update
53
+ use_iam_role
54
+ d = create_driver
55
+
56
+ aws_client_stub = Aws::RDS::Client.new(stub_responses: {
57
+ describe_db_log_files: {
58
+ describe_db_log_files: [
59
+ { log_file_name: 'server_audit.log', last_written: 123456789, size: 123 }
60
+ ],
61
+ marker: 'old_marker'
62
+ },
63
+ download_db_log_file_portion: {
64
+ log_file_data: "20250403 19:41:01,ip-1-1-1-1,service,1.2.3.4,12345678,1234567890,QUERY,test_db,'UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234',0,,",
65
+ marker: 'new_marker',
66
+ additional_data_pending: false
67
+ }
68
+ })
69
+
70
+ d.instance.instance_variable_set(:@rds, aws_client_stub)
71
+ d.instance.instance_variable_set(:@pos_info, { 'server_audit.log' => 'old_marker' })
72
+
73
+ d.run(timeout: 3, expect_emits: 1)
74
+ events = d.events
75
+
76
+ assert_equal(events[0][2]["log_file_name"], 'server_audit.log')
77
+ assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
78
+
79
+ # assert_equal(d.instance.instance_variable_get(:@pos_info)['server_audit.log'], 'new_marker')
80
+ end
81
+
82
+ def test_get_non_audit_log_files
83
+ use_iam_role
84
+ d = create_driver
85
+
86
+ aws_client_stub = Aws::RDS::Client.new(stub_responses: {
87
+ describe_db_log_files: {
88
+ describe_db_log_files: [
89
+ {
90
+ log_file_name: 'error.log',
91
+ last_written: 123456789,
92
+ size: 123
93
+ }
94
+ ],
95
+ marker: 'marker'
96
+ },
97
+ download_db_log_file_portion: {
98
+ log_file_data: "2025-03-21T00:00:04.275032Z 4071946 [Warning] [MY-010055] [Server] IP address '1.2.3.4' could not be resolved: Name or service not known",
99
+ marker: 'marker',
100
+ additional_data_pending: false
101
+ }
102
+ })
103
+
104
+ d.instance.instance_variable_set(:@rds, aws_client_stub)
105
+
106
+ # Simulate an older pos_last_written_timestamp
107
+ d.instance.instance_variable_set(:@pos_last_written_timestamp, 123456000)
108
+
109
+ d.run(timeout: 3, expect_emits: 1)
110
+
111
+ events = d.events
112
+
113
+ assert_equal(events[0][2]["log_file_name"], 'error.log')
114
+ assert_equal(events[0][2]["message"], "IP address '1.2.3.4' could not be resolved: Name or service not known")
115
+
116
+ # Ensure non-audit logs used `pos_last_written_timestamp`
117
+ assert_operator events[0][1].to_i, :>=, 123456000
118
+ end
119
+
120
+ def test_get_audit_log_files
121
+ use_iam_role
122
+ d = create_driver
123
+
124
+ aws_client_stub = Aws::RDS::Client.new(stub_responses: {
125
+ describe_db_log_files: {
126
+ describe_db_log_files: [
127
+ {
128
+ log_file_name: 'server_audit.log',
129
+ last_written: Time.now.to_i,
130
+ size: 123
131
+ }
132
+ ],
133
+ marker: 'marker'
134
+ },
135
+ download_db_log_file_portion: {
136
+ log_file_data: "20250403 19:41:01,ip-1-1-1-1,service,1.2.3.4,12345678,1234567890,QUERY,test_db,'UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234',0,,",
137
+ marker: 'marker',
138
+ additional_data_pending: false
139
+ }
140
+ })
141
+
142
+ d.instance.instance_variable_set(:@rds, aws_client_stub)
143
+
144
+ d.instance.instance_variable_set(:@pos_last_written_timestamp, (Time.now.to_i - (2*2000)))
145
+
146
+ test_start_time = Time.now.to_i
147
+ buffer_time = 5 * d.instance.refresh_interval * 20000
148
+
149
+ d.run(timeout: 3, expect_emits: 1)
150
+
151
+ events = d.events
152
+ assert_equal(events[0][2]["log_file_name"], 'server_audit.log')
153
+ assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
154
+
155
+ assert_operator events[0][1].to_i, :>=, (test_start_time - (buffer_time))
156
+ end
157
+ end
metadata ADDED
@@ -0,0 +1,170 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-rds-mysql-log
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Junaid Ali
8
+ bindir: bin
9
+ cert_chain: []
10
+ date: 2025-04-04 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: aws-sdk-ec2
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - "~>"
17
+ - !ruby/object:Gem::Version
18
+ version: '1.5'
19
+ type: :runtime
20
+ prerelease: false
21
+ version_requirements: !ruby/object:Gem::Requirement
22
+ requirements:
23
+ - - "~>"
24
+ - !ruby/object:Gem::Version
25
+ version: '1.5'
26
+ - !ruby/object:Gem::Dependency
27
+ name: aws-sdk-rds
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - "~>"
31
+ - !ruby/object:Gem::Version
32
+ version: '1.2'
33
+ type: :runtime
34
+ prerelease: false
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '1.2'
40
+ - !ruby/object:Gem::Dependency
41
+ name: fluentd
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: 0.14.0
47
+ - - "<"
48
+ - !ruby/object:Gem::Version
49
+ version: '2'
50
+ type: :runtime
51
+ prerelease: false
52
+ version_requirements: !ruby/object:Gem::Requirement
53
+ requirements:
54
+ - - ">="
55
+ - !ruby/object:Gem::Version
56
+ version: 0.14.0
57
+ - - "<"
58
+ - !ruby/object:Gem::Version
59
+ version: '2'
60
+ - !ruby/object:Gem::Dependency
61
+ name: bundler
62
+ requirement: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ - - "~>"
65
+ - !ruby/object:Gem::Version
66
+ version: '2.6'
67
+ type: :development
68
+ prerelease: false
69
+ version_requirements: !ruby/object:Gem::Requirement
70
+ requirements:
71
+ - - "~>"
72
+ - !ruby/object:Gem::Version
73
+ version: '2.6'
74
+ - !ruby/object:Gem::Dependency
75
+ name: rake
76
+ requirement: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - "~>"
79
+ - !ruby/object:Gem::Version
80
+ version: '13.2'
81
+ type: :development
82
+ prerelease: false
83
+ version_requirements: !ruby/object:Gem::Requirement
84
+ requirements:
85
+ - - "~>"
86
+ - !ruby/object:Gem::Version
87
+ version: '13.2'
88
+ - !ruby/object:Gem::Dependency
89
+ name: simplecov
90
+ requirement: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - "~>"
93
+ - !ruby/object:Gem::Version
94
+ version: '0.22'
95
+ type: :development
96
+ prerelease: false
97
+ version_requirements: !ruby/object:Gem::Requirement
98
+ requirements:
99
+ - - "~>"
100
+ - !ruby/object:Gem::Version
101
+ version: '0.22'
102
+ - !ruby/object:Gem::Dependency
103
+ name: test-unit
104
+ requirement: !ruby/object:Gem::Requirement
105
+ requirements:
106
+ - - "~>"
107
+ - !ruby/object:Gem::Version
108
+ version: '3.6'
109
+ type: :development
110
+ prerelease: false
111
+ version_requirements: !ruby/object:Gem::Requirement
112
+ requirements:
113
+ - - "~>"
114
+ - !ruby/object:Gem::Version
115
+ version: '3.6'
116
+ - !ruby/object:Gem::Dependency
117
+ name: webmock
118
+ requirement: !ruby/object:Gem::Requirement
119
+ requirements:
120
+ - - "~>"
121
+ - !ruby/object:Gem::Version
122
+ version: '3.2'
123
+ type: :development
124
+ prerelease: false
125
+ version_requirements: !ruby/object:Gem::Requirement
126
+ requirements:
127
+ - - "~>"
128
+ - !ruby/object:Gem::Version
129
+ version: '3.2'
130
+ description: fluentd plugin for Amazon RDS Mysql logs input
131
+ email:
132
+ - jonnie36@yahoo.com
133
+ executables: []
134
+ extensions: []
135
+ extra_rdoc_files: []
136
+ files:
137
+ - ".gitignore"
138
+ - Gemfile
139
+ - LICENSE
140
+ - README.md
141
+ - Rakefile
142
+ - fluent-plugin-rds-mysql-log.gemspec
143
+ - fluent.conf.sample
144
+ - lib/fluent/plugin/in_rds_mysql_log.rb
145
+ - test/helper.rb
146
+ - test/plugin/test_in_rds_mysql_log.rb
147
+ homepage: https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log.git
148
+ licenses:
149
+ - MIT
150
+ metadata: {}
151
+ rdoc_options: []
152
+ require_paths:
153
+ - lib
154
+ required_ruby_version: !ruby/object:Gem::Requirement
155
+ requirements:
156
+ - - ">="
157
+ - !ruby/object:Gem::Version
158
+ version: '0'
159
+ required_rubygems_version: !ruby/object:Gem::Requirement
160
+ requirements:
161
+ - - ">="
162
+ - !ruby/object:Gem::Version
163
+ version: '0'
164
+ requirements: []
165
+ rubygems_version: 3.6.2
166
+ specification_version: 4
167
+ summary: Amazon RDS Mysql logs input plugin
168
+ test_files:
169
+ - test/helper.rb
170
+ - test/plugin/test_in_rds_mysql_log.rb