fluent-plugin-rds-error-log 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: d315c53a55a2110c23be6faf35c81f687ad1d262
4
+ data.tar.gz: f1ed7bbb5a565d624754d690ae170d3dbc363377
5
+ SHA512:
6
+ metadata.gz: a37ff98498106ae1c4409bba3d0762460ec4c203fcbf2d20600bd5f82bd4a8f652fead484dbf2490f09926d3466f5708580197ccc31be6bbe7b622ed74c3c7c2
7
+ data.tar.gz: d5866eac496bf562e0635a2c85ab0b977d1757bdc108a4946fe5e00847cef9376dbd4162372980c910e66ed7bb618a9be63853ab5f292650182b57dd60f1abed
@@ -0,0 +1,3 @@
1
+ *.gem
2
+ fluent.conf
3
+
@@ -0,0 +1,5 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.2
4
+ - jruby
5
+ - rbx-2
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+ gem 'rake'
3
+ gem 'activerecord'
4
+ gem 'pg'
@@ -0,0 +1,35 @@
1
+ GEM
2
+ remote: https://rubygems.org/
3
+ specs:
4
+ activemodel (4.2.8)
5
+ activesupport (= 4.2.8)
6
+ builder (~> 3.1)
7
+ activerecord (4.2.8)
8
+ activemodel (= 4.2.8)
9
+ activesupport (= 4.2.8)
10
+ arel (~> 6.0)
11
+ activesupport (4.2.8)
12
+ i18n (~> 0.7)
13
+ minitest (~> 5.1)
14
+ thread_safe (~> 0.3, >= 0.3.4)
15
+ tzinfo (~> 1.1)
16
+ arel (6.0.4)
17
+ builder (3.2.3)
18
+ i18n (0.8.4)
19
+ minitest (5.10.2)
20
+ pg (0.20.0)
21
+ rake (12.0.0)
22
+ thread_safe (0.3.6)
23
+ tzinfo (1.2.3)
24
+ thread_safe (~> 0.1)
25
+
26
+ PLATFORMS
27
+ ruby
28
+
29
+ DEPENDENCIES
30
+ activerecord
31
+ pg
32
+ rake
33
+
34
+ BUNDLED WITH
35
+ 1.15.1
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2015 shinsaka
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,41 @@
1
+ # Amazon RDS for MySQL log input plugin for fluentd
2
+ (Pulled from forked version of fluent-plugin-mysql-log repo)
3
+
4
+ ## Overview
5
+ - Amazon Web Services RDS log input plugin for fluentd
6
+
7
+ ## Configuration
8
+
9
+ ```config
10
+ <source>
11
+ type rds_error_log
12
+ # required
13
+ region <region name>
14
+ db_instance_identifier <instance identifier>
15
+ # optional if you can IAM credentials
16
+ access_key_id <access_key>
17
+ secret_access_key <secret_access_key>
18
+ # optional
19
+ refresh_interval <interval number by second(default: 30)>
20
+ tag <tag name(default: rds-mysql.log>
21
+ pos_file <log getting position file(default: rds-mysql.log)>
22
+ </source>
23
+ ```
24
+
25
+ ### Example setting
26
+ ```config
27
+ <source>
28
+ type rds_error_log
29
+ region ap-northeast-1
30
+ db_instance_identifier test-mysql
31
+ access_key_id XXXXXXXXXXXXXXXXXXXX
32
+ secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
33
+ refresh_interval 30
34
+ tag mysql.log
35
+ pos_file /tmp/mysql-log-pos.dat
36
+ </source>
37
+
38
+ <match mysql.log>
39
+ type stdout
40
+ </match>
41
+ ```
@@ -0,0 +1,24 @@
1
+ lib = File.expand_path('../lib', __FILE__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+
4
+ Gem::Specification.new do |spec|
5
+ spec.name = "fluent-plugin-rds-error-log"
6
+ spec.version = "0.2.2"
7
+ spec.authors = ["Jen LaGrutta"]
8
+ spec.email = ["jen@tune.com"]
9
+ spec.summary = "Amazon RDS for Error/Audit log input plugin"
10
+ spec.description = "fluentd plugin for Amazon RDS for Error/Audit log input"
11
+ spec.homepage = "https://github.com/hasjenl/fluent-plugin-rds-error-log"
12
+ spec.license = "MIT"
13
+
14
+ spec.files = `git ls-files -z`.split("\x0")
15
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
16
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
17
+ spec.require_paths = ["lib"]
18
+
19
+ spec.add_dependency "fluentd", "~> 0"
20
+ spec.add_dependency "aws-sdk", "~> 2"
21
+ spec.add_dependency "myslog", "~> 0.0"
22
+
23
+ spec.add_development_dependency "bundler", "~> 1.7"
24
+ end
@@ -0,0 +1,14 @@
1
+ <source>
2
+ type rds_error_log
3
+ access_key_id XXXXXXXXXXXXXXXXXXXX
4
+ secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
5
+ region ap-northeast-1
6
+ db_instance_identifier test-mysql
7
+ refresh_interval 30
8
+ tag mysql.log
9
+ pos_file /error/mysql-error.log
10
+ </source>
11
+
12
+ <match mysql.log>
13
+ type stdout
14
+ </match>
@@ -0,0 +1,298 @@
1
+ require 'fluent/input'
2
+
3
+ class Fluent::RdsErrorLogInput < Fluent::Input
4
+ Fluent::Plugin.register_input("rds_error_log", self)
5
+
6
+ # To support log_level option implemented by Fluentd v0.10.43
7
+ unless method_defined?(:log)
8
+ define_method("log") { $log }
9
+ end
10
+
11
+ # Define `router` method of v0.12 to support v0.10 or earlier
12
+ unless method_defined?(:router)
13
+ define_method("router") { Fluent::Engine }
14
+ end
15
+
16
+ LOG_REGEXP = /^(?<time>\d{4}-\d{2}-\d{2} \d{2}\:\d{2}\:\d{2})( (?<pid>\d+))?( \[(?<message_level>[^\]]*?)\])? (?<message>.*)$/
17
+
18
+ config_param :access_key_id, :string, :default => nil
19
+ config_param :secret_access_key, :string, :default => nil
20
+ config_param :region, :string, :default => nil
21
+ config_param :db_instance_identifier, :string, :default => nil
22
+ config_param :pos_file, :string, :default => "fluent-plugin-rds-error-log-pos.dat"
23
+ config_param :refresh_interval, :integer, :default => 30
24
+ config_param :tag, :string, :default => "rds-mysql.log"
25
+ config_param :output_per_file, :bool, default: false
26
+ config_param :raw_output, :bool, default: false
27
+
28
+ def configure(conf)
29
+ super
30
+ require 'aws-sdk'
31
+
32
+ raise Fluent::ConfigError.new("region is required") unless @region
33
+ if !has_iam_role?
34
+ raise Fluent::ConfigError.new("access_key_id is required") if @access_key_id.nil?
35
+ raise Fluent::ConfigError.new("secret_access_key is required") if @secret_access_key.nil?
36
+ end
37
+ raise Fluent::ConfigError.new("db_instance_identifier is required") unless @db_instance_identifier
38
+ raise Fluent::ConfigError.new("pos_file is required") unless @pos_file
39
+ raise Fluent::ConfigError.new("refresh_interval is required") unless @refresh_interval
40
+ raise Fluent::ConfigError.new("tag is required") unless @tag
41
+ end
42
+
43
+ def start
44
+ super
45
+
46
+ # pos file touch
47
+ File.open(@pos_file, File::RDWR|File::CREAT).close
48
+
49
+ begin
50
+ options = {
51
+ :region => @region,
52
+ }
53
+ if @access_key_id && @secret_access_key
54
+ options[:access_key_id] = @access_key_id
55
+ options[:secret_access_key] = @secret_access_key
56
+ end
57
+ @rds = Aws::RDS::Client.new(options)
58
+ rescue => e
59
+ $log.warn "RDS Client error occurred: #{e.message}"
60
+ end
61
+
62
+ @loop = Coolio::Loop.new
63
+ timer_trigger = TimerWatcher.new(@refresh_interval, true, &method(:input))
64
+ timer_trigger.attach(@loop)
65
+ @thread = Thread.new(&method(:run))
66
+ end
67
+
68
+ def shutdown
69
+ super
70
+ @watcher.terminate
71
+ @thread.join
72
+ end
73
+
74
+ private
75
+
76
+ def run
77
+ @loop.run
78
+ end
79
+
80
+ def input
81
+ get_and_parse_posfile
82
+ log_files = get_logfile_list
83
+ get_logfile(log_files)
84
+ put_posfile
85
+ end
86
+
87
+ def has_iam_role?
88
+ begin
89
+ ec2 = Aws::EC2::Client.new(region: @region)
90
+ !ec2.config.credentials.nil?
91
+ rescue => e
92
+ $log.warn "EC2 Client error occurred: #{e.message}"
93
+ end
94
+ end
95
+
96
+ def get_and_parse_posfile
97
+ begin
98
+ # get & parse pos file
99
+ $log.debug "pos file get start"
100
+
101
+ pos_last_written_timestamp = 0
102
+ pos_info = {}
103
+ pos_log_hash = {}
104
+ File.open(@pos_file, File::RDONLY) do |file|
105
+ file.each_line do |line|
106
+
107
+ pos_match = /^timestamp: (\d+)$/.match(line)
108
+ if pos_match
109
+ pos_last_written_timestamp = pos_match[1].to_i
110
+ $log.debug "pos_last_written_timestamp: #{pos_last_written_timestamp}"
111
+ end
112
+
113
+ pos_match = /^(.+)\t(.+)\t(.+)$/.match(line)
114
+ if pos_match
115
+ pos_info[pos_match[1]] = pos_match[2]
116
+ pos_log_hash[pos_match[1]] = pos_match[3]
117
+ $log.debug "log_file: #{pos_match[1]}, marker: #{pos_match[2]}, hash: #{pos_match[3]}"
118
+ end
119
+ end
120
+ @pos_last_written_timestamp = pos_last_written_timestamp
121
+ @pos_info = pos_info
122
+ @pos_log_hash = pos_log_hash
123
+ end
124
+ rescue => e
125
+ $log.warn "pos file get and parse error occurred: #{e.message}"
126
+ end
127
+ end
128
+
129
+ def put_posfile
130
+ # pos file write
131
+ @pos_log_hash = @current_pos_log_hash
132
+ begin
133
+ $log.debug "pos file write"
134
+ File.open(@pos_file, File::WRONLY|File::TRUNC) do |file|
135
+ file.puts "timestamp: #{@pos_last_written_timestamp.to_s}"
136
+
137
+ @pos_info.each do |log_file_name, marker|
138
+ file.puts "#{log_file_name}\t#{marker}\t#{@pos_log_hash[log_file_name]}"
139
+ end
140
+ end
141
+ rescue => e
142
+ $log.warn "pos file write error occurred: #{e.message}"
143
+ end
144
+ end
145
+
146
+ def get_logfile_list
147
+ begin
148
+ $log.debug "get logfile-list from rds: db_instance_identifier=#{@db_instance_identifier}, pos_last_written_timestamp=#{@pos_last_written_timestamp}"
149
+ log_files = @rds.describe_db_log_files(
150
+ db_instance_identifier: @db_instance_identifier,
151
+ file_last_written: @pos_last_written_timestamp,
152
+ max_records: 10,
153
+ )
154
+
155
+ @current_pos_log_hash = {}
156
+ log_files.each do |log_file|
157
+ log_file.describe_db_log_files.each do |item|
158
+ log_file_name = item[:log_file_name]
159
+ @current_pos_log_hash[log_file_name] = item.hash
160
+ end
161
+ end
162
+
163
+ log_files
164
+ rescue => e
165
+ $log.warn "RDS Client describe_db_log_files error occurred: #{e.message}"
166
+ end
167
+ end
168
+
169
+ def rotated?(log_file)
170
+ utc_hour = (Time.now - 3600).utc.hour
171
+ hash_target = ""
172
+ case log_file
173
+ when "error/mysql-error.log" then
174
+ hash_target = "error/mysql-error-running.log"
175
+ else
176
+ hash_target = "#{log_file}.#{utc_hour}"
177
+ end
178
+ return @current_pos_log_hash[hash_target] && @pos_log_hash[hash_target] != @current_pos_log_hash[hash_target]
179
+ end
180
+
181
+ def get_logfile(log_files)
182
+ begin
183
+ log_files.each do |log_file|
184
+ log_file.describe_db_log_files.each do |item|
185
+ # save maximum written timestamp value
186
+ @pos_last_written_timestamp = item[:last_written] if @pos_last_written_timestamp < item[:last_written]
187
+
188
+ # log file download
189
+ log_file_name = item[:log_file_name]
190
+ next if log_file_name[%r{error/mysql-error-running.log}] || log_file_name[%r{slowquery/mysql-slowquery.log\.}] || log_file_name[%r{general/mysql-general.log\.}]
191
+ marker = @pos_info.has_key?(log_file_name) ? @pos_info[log_file_name] : "0"
192
+ marker = "0" if rotated?(log_file_name)
193
+
194
+ $log.debug "download log from rds: log_file_name=#{log_file_name}, marker=#{marker}"
195
+ logs = @rds.download_db_log_file_portion(
196
+ db_instance_identifier: @db_instance_identifier,
197
+ log_file_name: log_file_name,
198
+ marker: marker,
199
+ )
200
+ raw_records = get_logdata(logs)
201
+
202
+ #emit
203
+ parse_and_emit(raw_records, log_file_name) unless raw_records.nil?
204
+ end
205
+ end
206
+ rescue => e
207
+ $log.warn e.message
208
+ end
209
+ end
210
+
211
+ def get_logdata(logs)
212
+ log_file_name = logs.context.params[:log_file_name]
213
+ raw_records = []
214
+ begin
215
+ logs.each do |log|
216
+ # save got line's marker
217
+ @pos_info[log_file_name] = log.marker
218
+
219
+ raw_records += log.log_file_data.split("\n")
220
+ end
221
+ rescue => e
222
+ $log.warn e.message
223
+ end
224
+ return raw_records
225
+ end
226
+
227
+ def parse_and_emit(raw_records, log_file_name)
228
+ begin
229
+ $log.debug "raw_records.count: #{raw_records.count}"
230
+
231
+ record = {
232
+ "db_instance_identifier" => @db_instance_identifier,
233
+ "region" => @region,
234
+ "log_file_name" => log_file_name,
235
+ }
236
+ output_tag = @tag
237
+ output_tag += ".#{log_file_name.gsub(/\/|\./, '_')}" if @output_per_file
238
+
239
+ if log_file_name != "slowquery/mysql-slowquery.log"
240
+ raw_records.each do |raw_record|
241
+ $log.debug "raw_record=#{raw_record}"
242
+ line_match = LOG_REGEXP.match(raw_record)
243
+ next unless line_match
244
+
245
+ record["raw"] = raw_record if @raw_output
246
+ record["time"] = line_match[:time]
247
+ record["message"] = line_match[:message]
248
+ record["pid"] = line_match[:pid] if line_match[:pid]
249
+ record["message_level"] = line_match[:message_level] if line_match[:message_level]
250
+
251
+ router.emit(output_tag, Time.parse(line_match[:time] + ' +0000').to_i, record)
252
+ end
253
+ else
254
+ myslog = MySlog.new
255
+ myslog.divide(raw_records).each do |raw_record|
256
+ $log.debug "raw_record=#{raw_record}"
257
+ begin
258
+ raw = raw_record.join("\n") if @raw_output
259
+ record = record.merge(stringify_keys(myslog.parse_record(raw_record)))
260
+ record["raw"] = raw if @raw_output
261
+ if time = record.delete('date')
262
+ time = time.to_i
263
+ else
264
+ time = Time.now.to_i
265
+ end
266
+
267
+ router.emit(output_tag, time, record)
268
+ rescue => e
269
+ $log.warn e.message
270
+ end
271
+ end
272
+ end
273
+ rescue => e
274
+ $log.warn e.message
275
+ end
276
+ end
277
+
278
+ def stringify_keys(record)
279
+ result = {}
280
+ record.each_key do |key|
281
+ result[key.to_s] = record[key]
282
+ end
283
+ result
284
+ end
285
+
286
+ class TimerWatcher < Coolio::TimerWatcher
287
+ def initialize(interval, repeat, &callback)
288
+ require 'myslog'
289
+ @callback = callback
290
+ on_timer # first call
291
+ super(interval, repeat)
292
+ end
293
+
294
+ def on_timer
295
+ @callback.call
296
+ end
297
+ end
298
+ end
metadata ADDED
@@ -0,0 +1,109 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-rds-error-log
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.2.2
5
+ platform: ruby
6
+ authors:
7
+ - Jen LaGrutta
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-06-07 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: aws-sdk
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '2'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '2'
41
+ - !ruby/object:Gem::Dependency
42
+ name: myslog
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '0.0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '0.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: bundler
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.7'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.7'
69
+ description: fluentd plugin for Amazon RDS for Error/Audit log input
70
+ email:
71
+ - jen@tune.com
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - ".gitignore"
77
+ - ".travis.yml"
78
+ - Gemfile
79
+ - Gemfile.lock
80
+ - LICENSE.txt
81
+ - README.md
82
+ - fluent-plugin-rds-error-log.gemspec
83
+ - fluent.conf.sample
84
+ - lib/fluent/plugin/in_rds_error_log.rb
85
+ homepage: https://github.com/hasjenl/fluent-plugin-rds-error-log
86
+ licenses:
87
+ - MIT
88
+ metadata: {}
89
+ post_install_message:
90
+ rdoc_options: []
91
+ require_paths:
92
+ - lib
93
+ required_ruby_version: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - ">="
96
+ - !ruby/object:Gem::Version
97
+ version: '0'
98
+ required_rubygems_version: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: '0'
103
+ requirements: []
104
+ rubyforge_project:
105
+ rubygems_version: 2.4.8
106
+ signing_key:
107
+ specification_version: 4
108
+ summary: Amazon RDS for Error/Audit log input plugin
109
+ test_files: []