fluent-plugin-rds-pgsql-log 0.3.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/Rakefile +11 -0
- data/fluent-plugin-rds-pgsql-log.gemspec +5 -1
- data/lib/fluent/plugin/in_rds_pgsql_log.rb +16 -16
- data/test/helper.rb +20 -0
- data/test/plugin/test_in_rds_pgsql.rb +51 -0
- metadata +64 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fab744e32ec93e5bef3dea688e69b125f07469e7
|
4
|
+
data.tar.gz: 5aa94105f7c5f92d986aab90d929505fb8ee1d1a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 28b7c0f858271e41ae47d07a94e33bd0be57f5dfa04d9d96db2185f612680280d803f7593ac534aa29ce8cf7574d8cee5a4ae157a2321ad73fe5557e9cadf8ad
|
7
|
+
data.tar.gz: 262bdd80dcf2f10924c23d69d91bbe271b3692983bf943d2045accb6e65098893a64ae5564b4701816862c8fc8cdad04d15c9032ed9356119424f2201ca9a864
|
data/README.md
CHANGED
data/Rakefile
ADDED
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |spec|
|
5
5
|
spec.name = "fluent-plugin-rds-pgsql-log"
|
6
|
-
spec.version = "0.3.
|
6
|
+
spec.version = "0.3.1"
|
7
7
|
spec.authors = ["shinsaka"]
|
8
8
|
spec.email = ["shinx1265@gmail.com"]
|
9
9
|
spec.summary = "Amazon RDS for PostgreSQL log input plugin"
|
@@ -20,4 +20,8 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.add_dependency "aws-sdk", "~> 3"
|
21
21
|
|
22
22
|
spec.add_development_dependency "bundler", "~> 1.7"
|
23
|
+
spec.add_development_dependency "rake", "~> 12"
|
24
|
+
spec.add_development_dependency "test-unit", "~> 3.2"
|
25
|
+
spec.add_development_dependency "webmock", "~>3"
|
26
|
+
spec.add_development_dependency "simplecov", "~>0"
|
23
27
|
end
|
@@ -44,7 +44,7 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
|
|
44
44
|
end
|
45
45
|
@rds = Aws::RDS::Client.new(options)
|
46
46
|
rescue => e
|
47
|
-
|
47
|
+
log.warn "RDS Client error occurred: #{e.message}"
|
48
48
|
end
|
49
49
|
|
50
50
|
@loop = Coolio::Loop.new
|
@@ -77,14 +77,14 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
|
|
77
77
|
ec2 = Aws::EC2::Client.new(region: @region)
|
78
78
|
!ec2.config.credentials.nil?
|
79
79
|
rescue => e
|
80
|
-
|
80
|
+
log.warn "EC2 Client error occurred: #{e.message}"
|
81
81
|
end
|
82
82
|
end
|
83
83
|
|
84
84
|
def get_and_parse_posfile
|
85
85
|
begin
|
86
86
|
# get & parse pos file
|
87
|
-
|
87
|
+
log.debug "pos file get start"
|
88
88
|
|
89
89
|
pos_last_written_timestamp = 0
|
90
90
|
pos_info = {}
|
@@ -94,27 +94,27 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
|
|
94
94
|
pos_match = /^(\d+)$/.match(line)
|
95
95
|
if pos_match
|
96
96
|
pos_last_written_timestamp = pos_match[1].to_i
|
97
|
-
|
97
|
+
log.debug "pos_last_written_timestamp: #{pos_last_written_timestamp}"
|
98
98
|
end
|
99
99
|
|
100
100
|
pos_match = /^(.+)\t(.+)$/.match(line)
|
101
101
|
if pos_match
|
102
102
|
pos_info[pos_match[1]] = pos_match[2]
|
103
|
-
|
103
|
+
log.debug "log_file: #{pos_match[1]}, marker: #{pos_match[2]}"
|
104
104
|
end
|
105
105
|
end
|
106
106
|
@pos_last_written_timestamp = pos_last_written_timestamp
|
107
107
|
@pos_info = pos_info
|
108
108
|
end
|
109
109
|
rescue => e
|
110
|
-
|
110
|
+
log.warn "pos file get and parse error occurred: #{e.message}"
|
111
111
|
end
|
112
112
|
end
|
113
113
|
|
114
114
|
def put_posfile
|
115
115
|
# pos file write
|
116
116
|
begin
|
117
|
-
|
117
|
+
log.debug "pos file write"
|
118
118
|
File.open(@pos_file, File::WRONLY|File::TRUNC) do |file|
|
119
119
|
file.puts @pos_last_written_timestamp.to_s
|
120
120
|
|
@@ -123,20 +123,20 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
|
|
123
123
|
end
|
124
124
|
end
|
125
125
|
rescue => e
|
126
|
-
|
126
|
+
log.warn "pos file write error occurred: #{e.message}"
|
127
127
|
end
|
128
128
|
end
|
129
129
|
|
130
130
|
def get_logfile_list
|
131
131
|
begin
|
132
|
-
|
132
|
+
log.debug "get logfile-list from rds: db_instance_identifier=#{@db_instance_identifier}, pos_last_written_timestamp=#{@pos_last_written_timestamp}"
|
133
133
|
@rds.describe_db_log_files(
|
134
134
|
db_instance_identifier: @db_instance_identifier,
|
135
135
|
file_last_written: @pos_last_written_timestamp,
|
136
136
|
max_records: 10,
|
137
137
|
)
|
138
138
|
rescue => e
|
139
|
-
|
139
|
+
log.warn "RDS Client describe_db_log_files error occurred: #{e.message}"
|
140
140
|
end
|
141
141
|
end
|
142
142
|
|
@@ -151,7 +151,7 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
|
|
151
151
|
log_file_name = item[:log_file_name]
|
152
152
|
marker = @pos_info.has_key?(log_file_name) ? @pos_info[log_file_name] : "0"
|
153
153
|
|
154
|
-
|
154
|
+
log.debug "download log from rds: log_file_name=#{log_file_name}, marker=#{marker}"
|
155
155
|
logs = @rds.download_db_log_file_portion(
|
156
156
|
db_instance_identifier: @db_instance_identifier,
|
157
157
|
log_file_name: log_file_name,
|
@@ -164,7 +164,7 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
|
|
164
164
|
end
|
165
165
|
end
|
166
166
|
rescue => e
|
167
|
-
|
167
|
+
log.warn e.message
|
168
168
|
end
|
169
169
|
end
|
170
170
|
|
@@ -179,17 +179,17 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
|
|
179
179
|
raw_records += log.log_file_data.split("\n")
|
180
180
|
end
|
181
181
|
rescue => e
|
182
|
-
|
182
|
+
log.warn e.message
|
183
183
|
end
|
184
184
|
return raw_records
|
185
185
|
end
|
186
186
|
|
187
187
|
def parse_and_emit(raw_records, log_file_name)
|
188
188
|
begin
|
189
|
-
|
189
|
+
log.debug "raw_records.count: #{raw_records.count}"
|
190
190
|
record = nil
|
191
191
|
raw_records.each do |raw_record|
|
192
|
-
|
192
|
+
log.debug "raw_record=#{raw_record}"
|
193
193
|
line_match = LOG_REGEXP.match(raw_record)
|
194
194
|
|
195
195
|
unless line_match
|
@@ -215,7 +215,7 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
|
|
215
215
|
# emit last record
|
216
216
|
router.emit(@tag, Fluent::Engine.now, record) unless record.nil?
|
217
217
|
rescue => e
|
218
|
-
|
218
|
+
log.warn e.message
|
219
219
|
end
|
220
220
|
end
|
221
221
|
|
data/test/helper.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'bundler/setup'
|
2
|
+
require 'test/unit'
|
3
|
+
require 'webmock/test_unit'
|
4
|
+
require 'simplecov'
|
5
|
+
|
6
|
+
SimpleCov.start do
|
7
|
+
add_filter '/test/'
|
8
|
+
end
|
9
|
+
|
10
|
+
$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
|
11
|
+
$LOAD_PATH.unshift(__dir__)
|
12
|
+
require 'fluent/test'
|
13
|
+
require 'fluent/test/helpers'
|
14
|
+
require 'fluent/test/driver/input'
|
15
|
+
require 'fluent/plugin/in_rds_pgsql_log'
|
16
|
+
|
17
|
+
class Test::Unit::TestCase
|
18
|
+
include Fluent::Test::Helpers
|
19
|
+
extend Fluent::Test::Helpers
|
20
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
|
3
|
+
class RdsPgsqlLogInputTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
end
|
7
|
+
|
8
|
+
DEFAULT_CONFIG = {
|
9
|
+
access_key_id: 'dummy_access_key_id',
|
10
|
+
secret_access_key: 'dummy_secret_access_key',
|
11
|
+
region: 'ap-northeast-1',
|
12
|
+
db_instance_identifier: 'test-postgres-id',
|
13
|
+
refresh_interval: 30,
|
14
|
+
pos_file: 'pgsql-log-pos.dat',
|
15
|
+
}
|
16
|
+
|
17
|
+
def parse_config(conf = {})
|
18
|
+
''.tap{|s| conf.each { |k, v| s << "#{k} #{v}\n" } }
|
19
|
+
end
|
20
|
+
|
21
|
+
def create_driver(conf = DEFAULT_CONFIG)
|
22
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::RdsPgsqlLogInput).configure(parse_config conf)
|
23
|
+
end
|
24
|
+
|
25
|
+
def iam_info_url
|
26
|
+
'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
|
27
|
+
end
|
28
|
+
|
29
|
+
def use_iam_role
|
30
|
+
stub_request(:get, iam_info_url)
|
31
|
+
.to_return(status: [200, 'OK'], body: "hostname")
|
32
|
+
stub_request(:get, "#{iam_info_url}hostname")
|
33
|
+
.to_return(status: [200, 'OK'],
|
34
|
+
body: {
|
35
|
+
"AccessKeyId" => "dummy",
|
36
|
+
"SecretAccessKey" => "secret",
|
37
|
+
"Token" => "token"
|
38
|
+
}.to_json)
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_configure
|
42
|
+
use_iam_role
|
43
|
+
d = create_driver
|
44
|
+
assert_equal 'dummy_access_key_id', d.instance.access_key_id
|
45
|
+
assert_equal 'dummy_secret_access_key', d.instance.secret_access_key
|
46
|
+
assert_equal 'ap-northeast-1', d.instance.region
|
47
|
+
assert_equal 'test-postgres-id', d.instance.db_instance_identifier
|
48
|
+
assert_equal 'pgsql-log-pos.dat', d.instance.pos_file
|
49
|
+
assert_equal 30, d.instance.refresh_interval
|
50
|
+
end
|
51
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-rds-pgsql-log
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- shinsaka
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-07-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -58,6 +58,62 @@ dependencies:
|
|
58
58
|
- - "~>"
|
59
59
|
- !ruby/object:Gem::Version
|
60
60
|
version: '1.7'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: rake
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - "~>"
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '12'
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - "~>"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '12'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: test-unit
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - "~>"
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '3.2'
|
82
|
+
type: :development
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - "~>"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '3.2'
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: webmock
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - "~>"
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '3'
|
96
|
+
type: :development
|
97
|
+
prerelease: false
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - "~>"
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '3'
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
name: simplecov
|
105
|
+
requirement: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - "~>"
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
type: :development
|
111
|
+
prerelease: false
|
112
|
+
version_requirements: !ruby/object:Gem::Requirement
|
113
|
+
requirements:
|
114
|
+
- - "~>"
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: '0'
|
61
117
|
description: fluentd plugin for Amazon RDS for PostgreSQL log input
|
62
118
|
email:
|
63
119
|
- shinx1265@gmail.com
|
@@ -69,9 +125,12 @@ files:
|
|
69
125
|
- Gemfile
|
70
126
|
- LICENSE.txt
|
71
127
|
- README.md
|
128
|
+
- Rakefile
|
72
129
|
- fluent-plugin-rds-pgsql-log.gemspec
|
73
130
|
- fluent.conf.sample
|
74
131
|
- lib/fluent/plugin/in_rds_pgsql_log.rb
|
132
|
+
- test/helper.rb
|
133
|
+
- test/plugin/test_in_rds_pgsql.rb
|
75
134
|
homepage: https://github.com/shinsaka/fluent-plugin-rds-pgsql-log
|
76
135
|
licenses:
|
77
136
|
- MIT
|
@@ -96,4 +155,6 @@ rubygems_version: 2.6.11
|
|
96
155
|
signing_key:
|
97
156
|
specification_version: 4
|
98
157
|
summary: Amazon RDS for PostgreSQL log input plugin
|
99
|
-
test_files:
|
158
|
+
test_files:
|
159
|
+
- test/helper.rb
|
160
|
+
- test/plugin/test_in_rds_pgsql.rb
|