fluent-plugin-rds-mysql-log 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +5 -0
- data/Gemfile +5 -0
- data/LICENSE +21 -0
- data/README.md +90 -0
- data/Rakefile +14 -0
- data/fluent-plugin-rds-mysql-log.gemspec +30 -0
- data/fluent.conf.sample +14 -0
- data/lib/fluent/plugin/in_rds_mysql_log.rb +267 -0
- data/test/helper.rb +26 -0
- data/test/plugin/test_in_rds_mysql_log.rb +157 -0
- metadata +170 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 9b755fb7acda5fbc4825668d0351c4ed8b8259127a18027ff372ec1648ea6843
|
4
|
+
data.tar.gz: 4791ffd4877d9eaeba8b28a6674ff1365d096d44b961113c9f1a7ac27aed6115
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 9450bbed8aa34dfa3a1d4c06606b224d8c0cc11705292c1b6e11d02be963199fe5926c1875a2f34e94e999040c3152b305379a45083ae1fe5418212339c71550
|
7
|
+
data.tar.gz: ba76f33506086245ad6d1c2968466df20b1ca8541a18366ed1170333267bd3b99c860e955a092d547977f2764600ccbc9411d5187d57f85a7e75d0e0d22408d1
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) 2025 Junaid Ali
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
# fluent-plugin-rds-mysql-log
|
2
|
+
|
3
|
+
## Overview
|
4
|
+
- Amazon Web Services RDS log input plugin for fluentd
|
5
|
+
|
6
|
+
This plugin has been created to deal with audit and error logs of mysql.
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
$ fluentd-gem install fluent-plugin-rds-mysql-log
|
11
|
+
|
12
|
+
## AWS ELB Settings
|
13
|
+
- settings see: [Mysql Database Log Files](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_LogAccess.Concepts.Mysql.html)
|
14
|
+
|
15
|
+
## When SSL certification error
|
16
|
+
log:
|
17
|
+
```
|
18
|
+
SSL_connect returned=1 errno=0 state=SSLv3 read server certificate B: certificate verify failed
|
19
|
+
```
|
20
|
+
Do env setting follows:
|
21
|
+
```
|
22
|
+
SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
|
23
|
+
```
|
24
|
+
|
25
|
+
## Configuration
|
26
|
+
|
27
|
+
```config
|
28
|
+
<source>
|
29
|
+
type rds_mysql_log
|
30
|
+
# required
|
31
|
+
region <region name>
|
32
|
+
db_instance_identifier <instance identifier>
|
33
|
+
# optional if you can IAM credentials
|
34
|
+
access_key_id <access_key>
|
35
|
+
secret_access_key <secret_access_key>
|
36
|
+
# optional
|
37
|
+
refresh_interval <interval number by second(default: 30)>
|
38
|
+
tag <tag name(default: rds-mysql.log>
|
39
|
+
pos_file <log getting position file(default: rds-mysql.log)>
|
40
|
+
</source>
|
41
|
+
```
|
42
|
+
|
43
|
+
### Example setting
|
44
|
+
```config
|
45
|
+
<source>
|
46
|
+
type rds_mysql_log
|
47
|
+
region eu-central-1
|
48
|
+
db_instance_identifier test-mysql
|
49
|
+
access_key_id XXXXXXXXXXXXXXXXXXXX
|
50
|
+
secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
51
|
+
refresh_interval 30
|
52
|
+
tag mysql.log
|
53
|
+
pos_file /tmp/mysql-log-pos.dat
|
54
|
+
</source>
|
55
|
+
|
56
|
+
<match mysql.log>
|
57
|
+
type stdout
|
58
|
+
</match>
|
59
|
+
```
|
60
|
+
|
61
|
+
### json output example
|
62
|
+
```
|
63
|
+
Audit Logs:
|
64
|
+
|
65
|
+
{
|
66
|
+
"time" => "20250403 19:41:01",
|
67
|
+
"serverhost" => "ip-1-1-1-1",
|
68
|
+
"host" => "1.2.3.4",
|
69
|
+
"user" => "service",
|
70
|
+
"database" => "test_db",
|
71
|
+
"queryid" => "1234567890",
|
72
|
+
"connectionid" => "12345678",
|
73
|
+
"message" => "UPDATE table SET id=123, updated_at='2025-04-03 19:38:08.681797', is_weight_saved=1 WHERE table.id = 1234",
|
74
|
+
"return_code" => "0",
|
75
|
+
"log_file_name" => "server_audit.log"
|
76
|
+
}
|
77
|
+
|
78
|
+
Error Logs:
|
79
|
+
|
80
|
+
{
|
81
|
+
"time" => "2025-03-21T00:00:04.275032Z",
|
82
|
+
"thread_id" => "123456789",
|
83
|
+
"severity" => "Warning",
|
84
|
+
"error_code" => "MY-010055",
|
85
|
+
"subsystem" => "Server",
|
86
|
+
"message" => "IP address '1.2.3.4' could not be resolved: Name or service not known",
|
87
|
+
"log_file_name" => "error.log"
|
88
|
+
}
|
89
|
+
|
90
|
+
```
|
data/Rakefile
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
ENV['TZ'] = 'UTC'
|
5
|
+
require 'bundler/gem_tasks'
|
6
|
+
|
7
|
+
require 'rake/testtask'
|
8
|
+
Rake::TestTask.new(:test) do |test|
|
9
|
+
test.libs << 'lib' << 'test'
|
10
|
+
test.pattern = 'test/**/test_*.rb'
|
11
|
+
test.verbose = true
|
12
|
+
end
|
13
|
+
|
14
|
+
task default: :test
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'fluent-plugin-rds-mysql-log'
|
8
|
+
spec.version = '0.1.0'
|
9
|
+
spec.authors = ['Junaid Ali']
|
10
|
+
spec.email = ['jonnie36@yahoo.com']
|
11
|
+
spec.summary = 'Amazon RDS Mysql logs input plugin'
|
12
|
+
spec.description = 'fluentd plugin for Amazon RDS Mysql logs input'
|
13
|
+
spec.homepage = 'https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log.git'
|
14
|
+
spec.license = 'MIT'
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ['lib']
|
20
|
+
|
21
|
+
spec.add_dependency 'aws-sdk-ec2', '~> 1.5'
|
22
|
+
spec.add_dependency 'aws-sdk-rds', '~> 1.2'
|
23
|
+
spec.add_dependency 'fluentd', '>= 0.14.0', '< 2'
|
24
|
+
|
25
|
+
spec.add_development_dependency 'bundler', '~> 2.6'
|
26
|
+
spec.add_development_dependency 'rake', '~> 13.2'
|
27
|
+
spec.add_development_dependency 'simplecov', '~>0.22'
|
28
|
+
spec.add_development_dependency 'test-unit', '~> 3.6'
|
29
|
+
spec.add_development_dependency 'webmock', '~>3.2'
|
30
|
+
end
|
data/fluent.conf.sample
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
<source>
|
2
|
+
@type rds_mysql_log
|
3
|
+
access_key_id XXXXXXXXXXXXXXXXXXXX
|
4
|
+
secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
5
|
+
region eu-central-1
|
6
|
+
db_instance_identifier test-mysql
|
7
|
+
refresh_interval 30
|
8
|
+
tag mysql.log
|
9
|
+
pos_file /tmp/mysql-log-pos.dat
|
10
|
+
</source>
|
11
|
+
|
12
|
+
<match mysql.log>
|
13
|
+
@type stdout
|
14
|
+
</match>
|
@@ -0,0 +1,267 @@
|
|
1
|
+
require 'fluent/input'
|
2
|
+
require 'aws-sdk-ec2'
|
3
|
+
require 'aws-sdk-rds'
|
4
|
+
|
5
|
+
class Fluent::Plugin::RdsMysqlLogInput < Fluent::Plugin::Input
|
6
|
+
Fluent::Plugin.register_input('rds_mysql_log', self)
|
7
|
+
|
8
|
+
helpers :timer
|
9
|
+
|
10
|
+
LOG_REGEXP = /^(?:(?<audit_logs>(?<timestamp>(\d{8})\s(\d{2}:\d{2}:\d{2})?),(?<serverhost>[^,]+?),(?<username>[^,]+?),(?<host>[^,]+?),(?<connectionid>[^,]+?),(?<queryid>[^,]+?),(?<operation>[^,]+?),(?<database>[^,]+?),'(?<query>.*?)',(?<retcode>[0-9]?),(?:,)?)|(?<other_logs>(?<time>(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2}\.\d+Z)?)\s(?<thread_id>\d+?)\s\[(?<severity>([^\]]+)?)\]\s\[(?<error_code>([^\]]+)?)\]\s\[(?<subsystem>([^\]]+)?)\]\s((?<message>.+)?)))$/m
|
11
|
+
AUDIT_LOG_PATTERN = /server_audit\.log(\.\d+)?$/i
|
12
|
+
|
13
|
+
config_param :access_key_id, :string, :default => nil
|
14
|
+
config_param :secret_access_key, :string, :default => nil
|
15
|
+
config_param :region, :string, :default => nil
|
16
|
+
config_param :db_instance_identifier, :string, :default => nil
|
17
|
+
config_param :pos_file, :string, :default => "fluent-plugin-rds-mysql-log-pos.dat"
|
18
|
+
config_param :refresh_interval, :integer, :default => 30
|
19
|
+
config_param :tag, :string, :default => "rds-mysql.log"
|
20
|
+
|
21
|
+
def configure(conf)
|
22
|
+
super
|
23
|
+
|
24
|
+
raise Fluent::ConfigError.new("region is required") unless @region
|
25
|
+
if !has_iam_role?
|
26
|
+
raise Fluent::ConfigError.new("access_key_id is required") if @access_key_id.nil?
|
27
|
+
raise Fluent::ConfigError.new("secret_access_key is required") if @secret_access_key.nil?
|
28
|
+
end
|
29
|
+
raise Fluent::ConfigError.new("db_instance_identifier is required") unless @db_instance_identifier
|
30
|
+
raise Fluent::ConfigError.new("pos_file is required") unless @pos_file
|
31
|
+
raise Fluent::ConfigError.new("refresh_interval is required") unless @refresh_interval
|
32
|
+
raise Fluent::ConfigError.new("tag is required") unless @tag
|
33
|
+
|
34
|
+
begin
|
35
|
+
options = {
|
36
|
+
:region => @region,
|
37
|
+
}
|
38
|
+
if @access_key_id && @secret_access_key
|
39
|
+
options[:access_key_id] = @access_key_id
|
40
|
+
options[:secret_access_key] = @secret_access_key
|
41
|
+
end
|
42
|
+
@rds = Aws::RDS::Client.new(options)
|
43
|
+
rescue => e
|
44
|
+
log.warn "RDS Client error occurred: #{e.message}"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def start
|
49
|
+
super
|
50
|
+
|
51
|
+
# pos file touch
|
52
|
+
File.open(@pos_file, File::RDWR|File::CREAT).close
|
53
|
+
|
54
|
+
timer_execute(:poll_logs, @refresh_interval, repeat: true, &method(:input))
|
55
|
+
end
|
56
|
+
|
57
|
+
def shutdown
|
58
|
+
super
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def input
|
64
|
+
get_and_parse_posfile
|
65
|
+
log_files = get_logfile_list
|
66
|
+
get_logfile(log_files)
|
67
|
+
put_posfile
|
68
|
+
end
|
69
|
+
|
70
|
+
def has_iam_role?
|
71
|
+
begin
|
72
|
+
ec2 = Aws::EC2::Client.new(region: @region)
|
73
|
+
!ec2.config.credentials.nil?
|
74
|
+
rescue => e
|
75
|
+
log.warn "EC2 Client error occurred: #{e.message}"
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def get_and_parse_posfile
|
80
|
+
begin
|
81
|
+
# get & parse pos file
|
82
|
+
log.debug "pos file get start"
|
83
|
+
|
84
|
+
pos_last_written_timestamp = 0
|
85
|
+
pos_info = {}
|
86
|
+
File.open(@pos_file, File::RDONLY) do |file|
|
87
|
+
file.each_line do |line|
|
88
|
+
pos_match = /^(\d+)$/.match(line)
|
89
|
+
if pos_match
|
90
|
+
pos_last_written_timestamp = pos_match[1].to_i
|
91
|
+
log.debug "pos_last_written_timestamp: #{pos_last_written_timestamp}"
|
92
|
+
end
|
93
|
+
|
94
|
+
pos_match = /^(.+)\t(.+)$/.match(line)
|
95
|
+
if pos_match
|
96
|
+
pos_info[pos_match[1]] = pos_match[2]
|
97
|
+
p pos_info
|
98
|
+
log.debug "log_file: #{pos_match[1]}, marker: #{pos_match[2]}"
|
99
|
+
end
|
100
|
+
end
|
101
|
+
@pos_last_written_timestamp = pos_last_written_timestamp
|
102
|
+
@pos_info = pos_info
|
103
|
+
end
|
104
|
+
rescue => e
|
105
|
+
log.warn "pos file get and parse error occurred: #{e.message}"
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def put_posfile
|
110
|
+
# pos file write
|
111
|
+
begin
|
112
|
+
log.debug "pos file write"
|
113
|
+
File.open(@pos_file, File::WRONLY|File::TRUNC) do |file|
|
114
|
+
file.puts @pos_last_written_timestamp.to_s
|
115
|
+
|
116
|
+
@pos_info.each do |log_file_name, marker|
|
117
|
+
file.puts "#{log_file_name}\t#{marker}"
|
118
|
+
end
|
119
|
+
end
|
120
|
+
rescue => e
|
121
|
+
log.warn "pos file write error occurred: #{e.message}"
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def get_logfile_list
|
126
|
+
begin
|
127
|
+
log.debug "get logfile-list from rds: db_instance_identifier=#{@db_instance_identifier}, pos_last_written_timestamp=#{@pos_last_written_timestamp}"
|
128
|
+
|
129
|
+
# Separate audit logs from other logs
|
130
|
+
|
131
|
+
audit_logs_exist = @pos_info.keys.any? { |log_file_name| log_file_name =~ AUDIT_LOG_PATTERN }
|
132
|
+
|
133
|
+
file_last_written = if audit_logs_exist
|
134
|
+
# Use custom interval for audit logs
|
135
|
+
(Time.now.to_i - @refresh_interval) * 1000
|
136
|
+
else
|
137
|
+
# Use default timestamp for other logs
|
138
|
+
@pos_last_written_timestamp
|
139
|
+
end
|
140
|
+
|
141
|
+
@rds.describe_db_log_files(
|
142
|
+
db_instance_identifier: @db_instance_identifier,
|
143
|
+
file_last_written: file_last_written,
|
144
|
+
max_records: 10,
|
145
|
+
)
|
146
|
+
rescue => e
|
147
|
+
log.warn "RDS Client describe_db_log_files error occurred: #{e.message}"
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
def get_logfile(log_files)
|
152
|
+
begin
|
153
|
+
log_files.each do |log_file|
|
154
|
+
log_file.describe_db_log_files.each do |item|
|
155
|
+
# save maximum written timestamp value
|
156
|
+
@pos_last_written_timestamp = item[:last_written] if @pos_last_written_timestamp < item[:last_written]
|
157
|
+
|
158
|
+
# log file download
|
159
|
+
log_file_name = item[:log_file_name]
|
160
|
+
marker = @pos_info[log_file_name] || "0"
|
161
|
+
|
162
|
+
log.debug "download log from rds: log_file_name=#{log_file_name}, marker=#{marker}"
|
163
|
+
logs = @rds.download_db_log_file_portion(
|
164
|
+
db_instance_identifier: @db_instance_identifier,
|
165
|
+
log_file_name: log_file_name,
|
166
|
+
marker: marker,
|
167
|
+
)
|
168
|
+
|
169
|
+
raw_records = get_logdata(logs)
|
170
|
+
|
171
|
+
#emit
|
172
|
+
parse_and_emit(raw_records, log_file_name) unless raw_records.nil?
|
173
|
+
|
174
|
+
# Save new last_written timestamp **only for non-audit logs**
|
175
|
+
unless log_file_name =~ AUDIT_LOG_PATTERN
|
176
|
+
@pos_last_written_timestamp = item[:last_written] if @pos_last_written_timestamp < item[:last_written]
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
rescue => e
|
181
|
+
log.warn e.message
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
def get_logdata(logs)
|
186
|
+
log_file_name = logs.context.params[:log_file_name]
|
187
|
+
raw_records = []
|
188
|
+
begin
|
189
|
+
logs.each do |log|
|
190
|
+
# save got line's marker
|
191
|
+
@pos_info[log_file_name] = log.marker
|
192
|
+
|
193
|
+
raw_records += log.log_file_data.split("\n")
|
194
|
+
end
|
195
|
+
rescue => e
|
196
|
+
log.warn e.message
|
197
|
+
end
|
198
|
+
return raw_records
|
199
|
+
end
|
200
|
+
|
201
|
+
def event_time_of_row(record)
|
202
|
+
time = Time.parse(record["time"])
|
203
|
+
return Fluent::EventTime.from_time(time)
|
204
|
+
end
|
205
|
+
|
206
|
+
def parse_and_emit(raw_records, log_file_name)
|
207
|
+
begin
|
208
|
+
log.debug "raw_records.count: #{raw_records.count}"
|
209
|
+
|
210
|
+
record = nil
|
211
|
+
raw_records.each do |raw_record|
|
212
|
+
log.debug "raw_record=#{raw_record}"
|
213
|
+
line_match = LOG_REGEXP.match(raw_record)
|
214
|
+
|
215
|
+
unless line_match
|
216
|
+
# combine chain of log
|
217
|
+
record["message"] << "\n" + raw_record unless record.nil?
|
218
|
+
else
|
219
|
+
# emit before record
|
220
|
+
router.emit(@tag, event_time_of_row(record), record) unless record.nil?
|
221
|
+
|
222
|
+
# set a record
|
223
|
+
if line_match[:audit_logs]
|
224
|
+
record = {
|
225
|
+
"time" => line_match[:timestamp],
|
226
|
+
"serverhost" => line_match[:serverhost],
|
227
|
+
"host" => line_match[:host],
|
228
|
+
"user" => line_match[:username],
|
229
|
+
"database" => line_match[:database],
|
230
|
+
"queryid" => line_match[:queryid],
|
231
|
+
"connectionid" => line_match[:connectionid],
|
232
|
+
"message" => line_match[:query],
|
233
|
+
"return_code" => line_match[:retcode],
|
234
|
+
"log_file_name" => log_file_name,
|
235
|
+
}
|
236
|
+
else
|
237
|
+
record = {
|
238
|
+
"time" => line_match[:time],
|
239
|
+
"thread_id" => line_match[:thread_id],
|
240
|
+
"severity" => line_match[:severity],
|
241
|
+
"error_code" => line_match[:error_code],
|
242
|
+
"subsystem" => line_match[:subsystem],
|
243
|
+
"message" => line_match[:message],
|
244
|
+
"log_file_name" => log_file_name,
|
245
|
+
}
|
246
|
+
end
|
247
|
+
end
|
248
|
+
end
|
249
|
+
# emit last record
|
250
|
+
router.emit(@tag, event_time_of_row(record), record) unless record.nil?
|
251
|
+
rescue => e
|
252
|
+
log.warn e.message
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
256
|
+
class TimerWatcher < Coolio::TimerWatcher
|
257
|
+
def initialize(interval, repeat, &callback)
|
258
|
+
@callback = callback
|
259
|
+
on_timer # first call
|
260
|
+
super(interval, repeat)
|
261
|
+
end
|
262
|
+
|
263
|
+
def on_timer
|
264
|
+
@callback.call
|
265
|
+
end
|
266
|
+
end
|
267
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'test/unit'
|
5
|
+
require 'webmock/test_unit'
|
6
|
+
require 'simplecov'
|
7
|
+
|
8
|
+
SimpleCov.start do
|
9
|
+
add_filter '/test/'
|
10
|
+
end
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(__dir__)
|
14
|
+
require 'fluent/test'
|
15
|
+
require 'fluent/test/helpers'
|
16
|
+
require 'fluent/test/driver/input'
|
17
|
+
require 'fluent/plugin/in_rds_mysql_log'
|
18
|
+
|
19
|
+
module Test
|
20
|
+
module Unit
|
21
|
+
class TestCase
|
22
|
+
include Fluent::Test::Helpers
|
23
|
+
extend Fluent::Test::Helpers
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,157 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
|
3
|
+
class RdsMysqlLogInputTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
end
|
7
|
+
|
8
|
+
DEFAULT_CONFIG = {
|
9
|
+
access_key_id: 'dummy_access_key_id',
|
10
|
+
secret_access_key: 'dummy_secret_access_key',
|
11
|
+
region: 'ap-northeast-1',
|
12
|
+
db_instance_identifier: 'test-mysql-id',
|
13
|
+
refresh_interval: 2,
|
14
|
+
pos_file: 'mysql-log-pos.dat',
|
15
|
+
}
|
16
|
+
|
17
|
+
def parse_config(conf = {})
|
18
|
+
''.tap{|s| conf.each { |k, v| s << "#{k} #{v}\n" } }
|
19
|
+
end
|
20
|
+
|
21
|
+
def create_driver(conf = DEFAULT_CONFIG)
|
22
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::RdsMysqlLogInput).configure(parse_config conf)
|
23
|
+
end
|
24
|
+
|
25
|
+
def iam_info_url
|
26
|
+
'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
|
27
|
+
end
|
28
|
+
|
29
|
+
def use_iam_role
|
30
|
+
stub_request(:get, iam_info_url)
|
31
|
+
.to_return(status: [200, 'OK'], body: "hostname")
|
32
|
+
stub_request(:get, "#{iam_info_url}hostname")
|
33
|
+
.to_return(status: [200, 'OK'],
|
34
|
+
body: {
|
35
|
+
"AccessKeyId" => "dummy",
|
36
|
+
"SecretAccessKey" => "secret",
|
37
|
+
"Token" => "token"
|
38
|
+
}.to_json)
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_configure
|
42
|
+
use_iam_role
|
43
|
+
d = create_driver
|
44
|
+
assert_equal 'dummy_access_key_id', d.instance.access_key_id
|
45
|
+
assert_equal 'dummy_secret_access_key', d.instance.secret_access_key
|
46
|
+
assert_equal 'ap-northeast-1', d.instance.region
|
47
|
+
assert_equal 'test-mysql-id', d.instance.db_instance_identifier
|
48
|
+
assert_equal 'mysql-log-pos.dat', d.instance.pos_file
|
49
|
+
assert_equal 2, d.instance.refresh_interval
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_audit_log_marker_update
|
53
|
+
use_iam_role
|
54
|
+
d = create_driver
|
55
|
+
|
56
|
+
aws_client_stub = Aws::RDS::Client.new(stub_responses: {
|
57
|
+
describe_db_log_files: {
|
58
|
+
describe_db_log_files: [
|
59
|
+
{ log_file_name: 'server_audit.log', last_written: 123456789, size: 123 }
|
60
|
+
],
|
61
|
+
marker: 'old_marker'
|
62
|
+
},
|
63
|
+
download_db_log_file_portion: {
|
64
|
+
log_file_data: "20250403 19:41:01,ip-1-1-1-1,service,1.2.3.4,12345678,1234567890,QUERY,test_db,'UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234',0,,",
|
65
|
+
marker: 'new_marker',
|
66
|
+
additional_data_pending: false
|
67
|
+
}
|
68
|
+
})
|
69
|
+
|
70
|
+
d.instance.instance_variable_set(:@rds, aws_client_stub)
|
71
|
+
d.instance.instance_variable_set(:@pos_info, { 'server_audit.log' => 'old_marker' })
|
72
|
+
|
73
|
+
d.run(timeout: 3, expect_emits: 1)
|
74
|
+
events = d.events
|
75
|
+
|
76
|
+
assert_equal(events[0][2]["log_file_name"], 'server_audit.log')
|
77
|
+
assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
|
78
|
+
|
79
|
+
# assert_equal(d.instance.instance_variable_get(:@pos_info)['server_audit.log'], 'new_marker')
|
80
|
+
end
|
81
|
+
|
82
|
+
def test_get_non_audit_log_files
|
83
|
+
use_iam_role
|
84
|
+
d = create_driver
|
85
|
+
|
86
|
+
aws_client_stub = Aws::RDS::Client.new(stub_responses: {
|
87
|
+
describe_db_log_files: {
|
88
|
+
describe_db_log_files: [
|
89
|
+
{
|
90
|
+
log_file_name: 'error.log',
|
91
|
+
last_written: 123456789,
|
92
|
+
size: 123
|
93
|
+
}
|
94
|
+
],
|
95
|
+
marker: 'marker'
|
96
|
+
},
|
97
|
+
download_db_log_file_portion: {
|
98
|
+
log_file_data: "2025-03-21T00:00:04.275032Z 4071946 [Warning] [MY-010055] [Server] IP address '1.2.3.4' could not be resolved: Name or service not known",
|
99
|
+
marker: 'marker',
|
100
|
+
additional_data_pending: false
|
101
|
+
}
|
102
|
+
})
|
103
|
+
|
104
|
+
d.instance.instance_variable_set(:@rds, aws_client_stub)
|
105
|
+
|
106
|
+
# Simulate an older pos_last_written_timestamp
|
107
|
+
d.instance.instance_variable_set(:@pos_last_written_timestamp, 123456000)
|
108
|
+
|
109
|
+
d.run(timeout: 3, expect_emits: 1)
|
110
|
+
|
111
|
+
events = d.events
|
112
|
+
|
113
|
+
assert_equal(events[0][2]["log_file_name"], 'error.log')
|
114
|
+
assert_equal(events[0][2]["message"], "IP address '1.2.3.4' could not be resolved: Name or service not known")
|
115
|
+
|
116
|
+
# Ensure non-audit logs used `pos_last_written_timestamp`
|
117
|
+
assert_operator events[0][1].to_i, :>=, 123456000
|
118
|
+
end
|
119
|
+
|
120
|
+
def test_get_audit_log_files
|
121
|
+
use_iam_role
|
122
|
+
d = create_driver
|
123
|
+
|
124
|
+
aws_client_stub = Aws::RDS::Client.new(stub_responses: {
|
125
|
+
describe_db_log_files: {
|
126
|
+
describe_db_log_files: [
|
127
|
+
{
|
128
|
+
log_file_name: 'server_audit.log',
|
129
|
+
last_written: Time.now.to_i,
|
130
|
+
size: 123
|
131
|
+
}
|
132
|
+
],
|
133
|
+
marker: 'marker'
|
134
|
+
},
|
135
|
+
download_db_log_file_portion: {
|
136
|
+
log_file_data: "20250403 19:41:01,ip-1-1-1-1,service,1.2.3.4,12345678,1234567890,QUERY,test_db,'UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234',0,,",
|
137
|
+
marker: 'marker',
|
138
|
+
additional_data_pending: false
|
139
|
+
}
|
140
|
+
})
|
141
|
+
|
142
|
+
d.instance.instance_variable_set(:@rds, aws_client_stub)
|
143
|
+
|
144
|
+
d.instance.instance_variable_set(:@pos_last_written_timestamp, (Time.now.to_i - (2*2000)))
|
145
|
+
|
146
|
+
test_start_time = Time.now.to_i
|
147
|
+
buffer_time = 5 * d.instance.refresh_interval * 20000
|
148
|
+
|
149
|
+
d.run(timeout: 3, expect_emits: 1)
|
150
|
+
|
151
|
+
events = d.events
|
152
|
+
assert_equal(events[0][2]["log_file_name"], 'server_audit.log')
|
153
|
+
assert_equal(events[0][2]["message"], "UPDATE table SET id=123, updated_at=\'2025-04-03 19:38:08.681797\', is_weight_saved=1 WHERE table.id = 1234")
|
154
|
+
|
155
|
+
assert_operator events[0][1].to_i, :>=, (test_start_time - (buffer_time))
|
156
|
+
end
|
157
|
+
end
|
metadata
ADDED
@@ -0,0 +1,170 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-rds-mysql-log
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Junaid Ali
|
8
|
+
bindir: bin
|
9
|
+
cert_chain: []
|
10
|
+
date: 2025-04-04 00:00:00.000000000 Z
|
11
|
+
dependencies:
|
12
|
+
- !ruby/object:Gem::Dependency
|
13
|
+
name: aws-sdk-ec2
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - "~>"
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: '1.5'
|
19
|
+
type: :runtime
|
20
|
+
prerelease: false
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
22
|
+
requirements:
|
23
|
+
- - "~>"
|
24
|
+
- !ruby/object:Gem::Version
|
25
|
+
version: '1.5'
|
26
|
+
- !ruby/object:Gem::Dependency
|
27
|
+
name: aws-sdk-rds
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
29
|
+
requirements:
|
30
|
+
- - "~>"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '1.2'
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '1.2'
|
40
|
+
- !ruby/object:Gem::Dependency
|
41
|
+
name: fluentd
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 0.14.0
|
47
|
+
- - "<"
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: '2'
|
50
|
+
type: :runtime
|
51
|
+
prerelease: false
|
52
|
+
version_requirements: !ruby/object:Gem::Requirement
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: 0.14.0
|
57
|
+
- - "<"
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: '2'
|
60
|
+
- !ruby/object:Gem::Dependency
|
61
|
+
name: bundler
|
62
|
+
requirement: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - "~>"
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: '2.6'
|
67
|
+
type: :development
|
68
|
+
prerelease: false
|
69
|
+
version_requirements: !ruby/object:Gem::Requirement
|
70
|
+
requirements:
|
71
|
+
- - "~>"
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '2.6'
|
74
|
+
- !ruby/object:Gem::Dependency
|
75
|
+
name: rake
|
76
|
+
requirement: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - "~>"
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '13.2'
|
81
|
+
type: :development
|
82
|
+
prerelease: false
|
83
|
+
version_requirements: !ruby/object:Gem::Requirement
|
84
|
+
requirements:
|
85
|
+
- - "~>"
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
version: '13.2'
|
88
|
+
- !ruby/object:Gem::Dependency
|
89
|
+
name: simplecov
|
90
|
+
requirement: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - "~>"
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0.22'
|
95
|
+
type: :development
|
96
|
+
prerelease: false
|
97
|
+
version_requirements: !ruby/object:Gem::Requirement
|
98
|
+
requirements:
|
99
|
+
- - "~>"
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0.22'
|
102
|
+
- !ruby/object:Gem::Dependency
|
103
|
+
name: test-unit
|
104
|
+
requirement: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - "~>"
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '3.6'
|
109
|
+
type: :development
|
110
|
+
prerelease: false
|
111
|
+
version_requirements: !ruby/object:Gem::Requirement
|
112
|
+
requirements:
|
113
|
+
- - "~>"
|
114
|
+
- !ruby/object:Gem::Version
|
115
|
+
version: '3.6'
|
116
|
+
- !ruby/object:Gem::Dependency
|
117
|
+
name: webmock
|
118
|
+
requirement: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - "~>"
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '3.2'
|
123
|
+
type: :development
|
124
|
+
prerelease: false
|
125
|
+
version_requirements: !ruby/object:Gem::Requirement
|
126
|
+
requirements:
|
127
|
+
- - "~>"
|
128
|
+
- !ruby/object:Gem::Version
|
129
|
+
version: '3.2'
|
130
|
+
description: fluentd plugin for Amazon RDS Mysql logs input
|
131
|
+
email:
|
132
|
+
- jonnie36@yahoo.com
|
133
|
+
executables: []
|
134
|
+
extensions: []
|
135
|
+
extra_rdoc_files: []
|
136
|
+
files:
|
137
|
+
- ".gitignore"
|
138
|
+
- Gemfile
|
139
|
+
- LICENSE
|
140
|
+
- README.md
|
141
|
+
- Rakefile
|
142
|
+
- fluent-plugin-rds-mysql-log.gemspec
|
143
|
+
- fluent.conf.sample
|
144
|
+
- lib/fluent/plugin/in_rds_mysql_log.rb
|
145
|
+
- test/helper.rb
|
146
|
+
- test/plugin/test_in_rds_mysql_log.rb
|
147
|
+
homepage: https://github.com/iamjunaidali/fluent-plugin-rds-mysql-log.git
|
148
|
+
licenses:
|
149
|
+
- MIT
|
150
|
+
metadata: {}
|
151
|
+
rdoc_options: []
|
152
|
+
require_paths:
|
153
|
+
- lib
|
154
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - ">="
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '0'
|
159
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
160
|
+
requirements:
|
161
|
+
- - ">="
|
162
|
+
- !ruby/object:Gem::Version
|
163
|
+
version: '0'
|
164
|
+
requirements: []
|
165
|
+
rubygems_version: 3.6.2
|
166
|
+
specification_version: 4
|
167
|
+
summary: Amazon RDS Mysql logs input plugin
|
168
|
+
test_files:
|
169
|
+
- test/helper.rb
|
170
|
+
- test/plugin/test_in_rds_mysql_log.rb
|