fluent-plugin-postgresql-csvlog 0.3.2 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.gitlab-ci.yml +27 -3
- data/README.md +17 -0
- data/docker-compose.yml +26 -8
- data/example-fluentd.conf +30 -1
- data/fluent-plugin-postgresql-csvlog.gemspec +1 -1
- data/lib/fluent/plugin/filter_marginalia.rb +3 -67
- data/lib/fluent/plugin/in_pg_stat_activity.rb +118 -0
- data/lib/fluent/plugin/in_pg_stat_statements.rb +59 -77
- data/lib/fluent/plugin/marginalia_extractor.rb +73 -0
- data/lib/fluent/plugin/polling_pg_input_plugin.rb +100 -0
- data/test/helper.rb +2 -0
- data/test/plugin/itest_in_pg_stat_activity.rb +115 -0
- data/test/plugin/itest_in_pg_stat_statements.rb +3 -3
- data/test/plugin/test_in_pg_stat_activity.rb +223 -0
- data/test/plugin/test_in_pg_stat_statements.rb +64 -10
- data/test/plugin/test_marginalia_extractor.rb +103 -0
- data/test/verify-docker-compose.sh +32 -0
- metadata +18 -6
@@ -0,0 +1,100 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'fluent/plugin/input'
|
4
|
+
require 'pg'
|
5
|
+
require 'pg_query'
|
6
|
+
|
7
|
+
module Fluent::Plugin
|
8
|
+
# PollingPostgresInputPlugin is intended to be used as an base class
|
9
|
+
# for input plugins that poll postgres.
|
10
|
+
#
|
11
|
+
# Child classes should implement the `on_poll` method
|
12
|
+
class PollingPostgresInputPlugin < Input
|
13
|
+
desc 'PostgreSQL host'
|
14
|
+
config_param :host, :string
|
15
|
+
|
16
|
+
desc 'RDBMS port (default: 5432)'
|
17
|
+
config_param :port, :integer, default: 5432
|
18
|
+
|
19
|
+
desc 'login user name'
|
20
|
+
config_param :username, :string, default: nil
|
21
|
+
|
22
|
+
desc 'postgres db'
|
23
|
+
config_param :dbname, :string, default: nil
|
24
|
+
|
25
|
+
desc 'login password'
|
26
|
+
config_param :password, :string, default: nil, secret: true
|
27
|
+
|
28
|
+
# See https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE
|
29
|
+
# for options
|
30
|
+
desc 'postgres sslmode'
|
31
|
+
config_param :sslmode, :string, default: 'prefer'
|
32
|
+
|
33
|
+
desc 'tag'
|
34
|
+
config_param :tag, :string, default: nil
|
35
|
+
|
36
|
+
desc 'interval in second to run query'
|
37
|
+
config_param :interval, :time, default: 300
|
38
|
+
|
39
|
+
def start
|
40
|
+
@stop_flag = false
|
41
|
+
@thread = Thread.new(&method(:thread_main))
|
42
|
+
end
|
43
|
+
|
44
|
+
# Fluentd shutdown method, called to terminate and cleanup plugin
|
45
|
+
def shutdown
|
46
|
+
@stop_flag = true
|
47
|
+
|
48
|
+
# Interrupt thread and wait for it to finish
|
49
|
+
Thread.new { @thread.run } if @thread
|
50
|
+
@thread.join
|
51
|
+
end
|
52
|
+
|
53
|
+
# Main polling loop on thread
|
54
|
+
def thread_main
|
55
|
+
until @stop_flag
|
56
|
+
sleep @interval
|
57
|
+
break if @stop_flag
|
58
|
+
|
59
|
+
begin
|
60
|
+
on_poll
|
61
|
+
rescue StandardError => e
|
62
|
+
log.error 'unexpected error', error: e.message, error_class: e.class
|
63
|
+
log.error_backtrace
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
protected
|
69
|
+
|
70
|
+
# Child-classes should implement this method
|
71
|
+
def on_poll
|
72
|
+
raise 'on_poll must be implemented by descendents of PollingPostgresInputPlugin'
|
73
|
+
end
|
74
|
+
|
75
|
+
# Since this query is very infrequent, and it may be communicating directly
|
76
|
+
# with postgres without pgbouncer, don't use a persistent connection and
|
77
|
+
# ensure that it is properly closed
|
78
|
+
def with_connection(&block)
|
79
|
+
conn = PG.connect(
|
80
|
+
host: @host,
|
81
|
+
dbname: @dbname,
|
82
|
+
sslmode: @sslmode,
|
83
|
+
user: @username,
|
84
|
+
password: @password
|
85
|
+
)
|
86
|
+
|
87
|
+
map = PG::BasicTypeMapForResults.new(conn)
|
88
|
+
map.default_type_map = PG::TypeMapAllStrings.new
|
89
|
+
|
90
|
+
conn.type_map_for_results = map
|
91
|
+
|
92
|
+
begin
|
93
|
+
block.call(conn)
|
94
|
+
ensure
|
95
|
+
# Always close the connection
|
96
|
+
conn.finish
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
data/test/helper.rb
CHANGED
@@ -16,4 +16,6 @@ Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
|
16
16
|
require 'fluent/plugin/filter_postgresql_slowlog'
|
17
17
|
require 'fluent/plugin/filter_postgresql_redactor'
|
18
18
|
require 'fluent/plugin/filter_marginalia'
|
19
|
+
require 'fluent/plugin/marginalia_extractor'
|
19
20
|
require 'fluent/plugin/in_pg_stat_statements'
|
21
|
+
require 'fluent/plugin/in_pg_stat_activity'
|
@@ -0,0 +1,115 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../helper'
|
4
|
+
|
5
|
+
class PgStatActivityInputIntegrationTest < Test::Unit::TestCase
|
6
|
+
# The defaults values work with the configuration in .gitlab-ci.yml on the postgres service
|
7
|
+
# Override with env vars for local development
|
8
|
+
HOST = ENV.fetch('PG_TEST_HOST', 'postgres')
|
9
|
+
USERNAME = ENV.fetch('PG_TEST_USER', 'testuser')
|
10
|
+
PASSWORD = ENV.fetch('PG_TEST_PASSWORD', 'testpass')
|
11
|
+
|
12
|
+
def setup
|
13
|
+
Fluent::Test.setup
|
14
|
+
|
15
|
+
@conn = PG.connect(
|
16
|
+
host: HOST,
|
17
|
+
user: USERNAME,
|
18
|
+
password: PASSWORD
|
19
|
+
)
|
20
|
+
end
|
21
|
+
|
22
|
+
def teardown
|
23
|
+
@conn&.finish
|
24
|
+
end
|
25
|
+
|
26
|
+
VALID_CONFIG = %(
|
27
|
+
tag postgres.pg_stat_statements
|
28
|
+
host #{HOST}
|
29
|
+
username #{USERNAME}
|
30
|
+
password #{PASSWORD}
|
31
|
+
interval 1
|
32
|
+
)
|
33
|
+
|
34
|
+
INVALID_CONFIG = %(
|
35
|
+
host 'invalid_host.dne'
|
36
|
+
port 1234
|
37
|
+
username #{USERNAME}
|
38
|
+
password #{PASSWORD}
|
39
|
+
interval 1
|
40
|
+
)
|
41
|
+
|
42
|
+
def create_driver(config)
|
43
|
+
Fluent::Test::InputTestDriver.new(Fluent::Plugin::PgStatActivityInput).configure(config)
|
44
|
+
end
|
45
|
+
|
46
|
+
sub_test_case 'configuration' do
|
47
|
+
test 'connects' do
|
48
|
+
d = create_driver(VALID_CONFIG)
|
49
|
+
|
50
|
+
emits = []
|
51
|
+
# wait 50 * 0.05, "see fluentd/lib/fluent/test/base.rb:79 num_waits.times { sleep 0.05 }
|
52
|
+
d.run(num_waits = 50) do
|
53
|
+
emits = d.emits
|
54
|
+
end
|
55
|
+
|
56
|
+
assert_false emits.empty?
|
57
|
+
end
|
58
|
+
|
59
|
+
# Why do we have this test? If postgres is still starting up, we don't want to cause the
|
60
|
+
# the fluentd configuration to fail. We would rather retry until we get a connection
|
61
|
+
test 'connects for an invalid config' do
|
62
|
+
d = create_driver(INVALID_CONFIG)
|
63
|
+
|
64
|
+
emits = []
|
65
|
+
# wait 50 * 0.05, "see fluentd/lib/fluent/test/base.rb:79 num_waits.times { sleep 0.05 }
|
66
|
+
d.run(num_waits = 50) do
|
67
|
+
emits = d.emits
|
68
|
+
end
|
69
|
+
|
70
|
+
assert_true emits.empty?
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
sub_test_case 'execution' do
|
75
|
+
test 'connects' do
|
76
|
+
d = create_driver(VALID_CONFIG)
|
77
|
+
|
78
|
+
emits = []
|
79
|
+
# wait 50 * 0.05, "see fluentd/lib/fluent/test/base.rb:79 num_waits.times { sleep 0.05 }
|
80
|
+
d.run(num_waits = 50) do
|
81
|
+
emits = d.emits
|
82
|
+
end
|
83
|
+
|
84
|
+
first_with_query = emits.find do |event|
|
85
|
+
record = event[2]
|
86
|
+
|
87
|
+
record['usename'] == USERNAME &&
|
88
|
+
!record['datid'].nil? &&
|
89
|
+
!record['query'].nil? &&
|
90
|
+
record['state'] == 'active'
|
91
|
+
end
|
92
|
+
|
93
|
+
assert_false first_with_query.nil?
|
94
|
+
record = first_with_query[2]
|
95
|
+
|
96
|
+
assert_false record['datname'].nil?
|
97
|
+
assert_false record['pid'].nil?
|
98
|
+
assert_false record['usesysid'].nil?
|
99
|
+
assert_false record['application_name'].nil?
|
100
|
+
assert_false record['client_addr'].nil?
|
101
|
+
assert_false record['client_port'].nil?
|
102
|
+
assert_false record['xact_start'].nil?
|
103
|
+
assert_false record['xact_age_s'].nil?
|
104
|
+
assert_false record['xact_start'].nil?
|
105
|
+
assert_false record['xact_age_s'].nil?
|
106
|
+
assert_false record['query_start'].nil?
|
107
|
+
assert_false record['query_age_s'].nil?
|
108
|
+
assert_false record['state_change'].nil?
|
109
|
+
assert_false record['state_age_s'].nil?
|
110
|
+
assert_false record['query_length'].nil?
|
111
|
+
assert_false record['query'].nil?
|
112
|
+
assert_false record['fingerprint'].nil?
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -98,8 +98,7 @@ class PgStatStatementsInputIntegrationTest < Test::Unit::TestCase
|
|
98
98
|
expected_record = {
|
99
99
|
'fingerprint' => '8a6e9896bd9048a2',
|
100
100
|
'query' => 'SELECT * FROM pg_stat_statements ORDER BY queryid LIMIT $1',
|
101
|
-
'query_length' => 58
|
102
|
-
'queryid' => 3_239_318_621_761_098_074
|
101
|
+
'query_length' => 58
|
103
102
|
}
|
104
103
|
known_statement_event = emits.find do |event|
|
105
104
|
record = event[2]
|
@@ -114,7 +113,8 @@ class PgStatStatementsInputIntegrationTest < Test::Unit::TestCase
|
|
114
113
|
assert_equal 'postgres.pg_stat_statements', tag
|
115
114
|
assert_equal expected_record['fingerprint'], record['fingerprint']
|
116
115
|
assert_equal expected_record['query_length'], record['query_length']
|
117
|
-
assert_true
|
116
|
+
assert_true record.include? 'queryid'
|
117
|
+
assert_true record['queryid'].is_a? String
|
118
118
|
end
|
119
119
|
end
|
120
120
|
end
|
@@ -0,0 +1,223 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../helper'
|
4
|
+
require 'date'
|
5
|
+
|
6
|
+
class PgStatActivityInputTest < Test::Unit::TestCase
|
7
|
+
def setup
|
8
|
+
Fluent::Test.setup
|
9
|
+
end
|
10
|
+
|
11
|
+
CONFIG = %(
|
12
|
+
tag postgres.pg_stat_activity
|
13
|
+
host localhost
|
14
|
+
port 1234
|
15
|
+
dbname gitlab
|
16
|
+
sslmode require
|
17
|
+
username moo
|
18
|
+
password secret
|
19
|
+
interval 600
|
20
|
+
fingerprint_key fingerprint
|
21
|
+
)
|
22
|
+
|
23
|
+
def create_driver
|
24
|
+
Fluent::Test::InputTestDriver.new(Fluent::Plugin::PgStatActivityInput).configure(CONFIG)
|
25
|
+
end
|
26
|
+
|
27
|
+
sub_test_case 'configuration' do
|
28
|
+
test 'basic configuration' do
|
29
|
+
d = create_driver
|
30
|
+
|
31
|
+
assert_equal 'postgres.pg_stat_activity', d.instance.tag
|
32
|
+
assert_equal 'localhost', d.instance.host
|
33
|
+
assert_equal 1234, d.instance.port
|
34
|
+
assert_equal 'gitlab', d.instance.dbname
|
35
|
+
assert_equal 'require', d.instance.sslmode
|
36
|
+
assert_equal 'moo', d.instance.username
|
37
|
+
assert_equal 'secret', d.instance.password
|
38
|
+
assert_equal 600, d.instance.interval
|
39
|
+
assert_equal 'fingerprint', d.instance.fingerprint_key
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
sub_test_case 'execution' do
|
44
|
+
test 'sql' do
|
45
|
+
d = create_driver
|
46
|
+
row = {
|
47
|
+
'datid' => 16384,
|
48
|
+
'datname' => 'testuser',
|
49
|
+
'pid' => 376,
|
50
|
+
'usesysid' => 10,
|
51
|
+
'usename' => 'testuser',
|
52
|
+
'application_name' => 'psql',
|
53
|
+
'client_addr' => '172.17.0.1',
|
54
|
+
'client_hostname' => nil,
|
55
|
+
'client_port' => 60182,
|
56
|
+
'xact_start' => Time.parse('2021-07-23 12:55:25 +0000'),
|
57
|
+
'xact_age_s' => 0.001884,
|
58
|
+
'query_start' => Time.parse('2021-07-23 12:55:25 +0000'),
|
59
|
+
'query_age_s' => 0.001894,
|
60
|
+
'state_change' => Time.parse('2021-07-23 12:55:25 +0000'),
|
61
|
+
'state_age_s' => 0.001894,
|
62
|
+
'state' => 'active',
|
63
|
+
'query' => "SELECT * FROM users WHERE user_secret = 's3cr3t'"
|
64
|
+
}
|
65
|
+
|
66
|
+
record = d.instance.record_for_row(row)
|
67
|
+
|
68
|
+
expected = {
|
69
|
+
'application_name' => 'psql',
|
70
|
+
'client_addr' => '172.17.0.1',
|
71
|
+
'client_hostname' => nil,
|
72
|
+
'client_port' => 60182,
|
73
|
+
'datid' => 16384,
|
74
|
+
'datname' => 'testuser',
|
75
|
+
'fingerprint' => '5c4a61e156c7d822',
|
76
|
+
'pid' => 376,
|
77
|
+
'query' => 'SELECT * FROM users WHERE user_secret = $1', # NOTE: secret removed
|
78
|
+
'query_age_s' => 0.001894,
|
79
|
+
'query_length' => 48,
|
80
|
+
'query_start' => '2021-07-23T12:55:25.000+00:00',
|
81
|
+
'state' => 'active',
|
82
|
+
'state_age_s' => 0.001894,
|
83
|
+
'state_change' => '2021-07-23T12:55:25.000+00:00',
|
84
|
+
'usename' => 'testuser',
|
85
|
+
'usesysid' => 10,
|
86
|
+
'xact_age_s' => 0.001884,
|
87
|
+
'xact_start' => '2021-07-23T12:55:25.000+00:00'
|
88
|
+
}
|
89
|
+
|
90
|
+
assert_equal expected, record
|
91
|
+
end
|
92
|
+
|
93
|
+
# This test mostly checks that the code is nil safe
|
94
|
+
test 'nil query' do
|
95
|
+
d = create_driver
|
96
|
+
record = d.instance.record_for_row({})
|
97
|
+
|
98
|
+
expected = {
|
99
|
+
'application_name' => nil,
|
100
|
+
'client_addr' => nil,
|
101
|
+
'client_hostname' => nil,
|
102
|
+
'client_port' => nil,
|
103
|
+
'datid' => nil,
|
104
|
+
'datname' => nil,
|
105
|
+
'pid' => nil,
|
106
|
+
'query' => nil,
|
107
|
+
'query_age_s' => nil,
|
108
|
+
'query_length' => nil,
|
109
|
+
'query_start' => nil,
|
110
|
+
'state' => nil,
|
111
|
+
'state_age_s' => nil,
|
112
|
+
'state_change' => nil,
|
113
|
+
'usename' => nil,
|
114
|
+
'usesysid' => nil,
|
115
|
+
'xact_age_s' => nil,
|
116
|
+
'xact_start' => nil
|
117
|
+
}
|
118
|
+
|
119
|
+
assert_equal expected, record
|
120
|
+
end
|
121
|
+
|
122
|
+
test 'unparseable sql' do
|
123
|
+
d = create_driver
|
124
|
+
row = {
|
125
|
+
'datid' => 16384,
|
126
|
+
'datname' => 'testuser',
|
127
|
+
'pid' => 376,
|
128
|
+
'usesysid' => 10,
|
129
|
+
'usename' => 'testuser',
|
130
|
+
'application_name' => 'psql',
|
131
|
+
'client_addr' => '172.17.0.1',
|
132
|
+
'client_hostname' => nil,
|
133
|
+
'client_port' => 60182,
|
134
|
+
'xact_start' => Time.parse('2021-07-23 12:55:25 +0000'),
|
135
|
+
'xact_age_s' => 0.001884,
|
136
|
+
'query_start' => Time.parse('2021-07-23 12:55:25 +0000'),
|
137
|
+
'query_age_s' => 0.001894,
|
138
|
+
'state_change' => Time.parse('2021-07-23 12:55:25 +0000'),
|
139
|
+
'state_age_s' => 0.001894,
|
140
|
+
'state' => 'active',
|
141
|
+
'query' => "SELECT * FROM users WHERE user_se="
|
142
|
+
}
|
143
|
+
|
144
|
+
record = d.instance.record_for_row(row)
|
145
|
+
|
146
|
+
expected = {
|
147
|
+
'application_name' => 'psql',
|
148
|
+
'client_addr' => '172.17.0.1',
|
149
|
+
'client_hostname' => nil,
|
150
|
+
'client_port' => 60182,
|
151
|
+
'datid' => 16384,
|
152
|
+
'datname' => 'testuser',
|
153
|
+
'pid' => 376,
|
154
|
+
'query' => nil,
|
155
|
+
'query_age_s' => 0.001894,
|
156
|
+
'query_length' => 34,
|
157
|
+
'query_start' => '2021-07-23T12:55:25.000+00:00',
|
158
|
+
'query_unparseable' => true,
|
159
|
+
'state' => 'active',
|
160
|
+
'state_age_s' => 0.001894,
|
161
|
+
'state_change' => '2021-07-23T12:55:25.000+00:00',
|
162
|
+
'usename' => 'testuser',
|
163
|
+
'usesysid' => 10,
|
164
|
+
'xact_age_s' => 0.001884,
|
165
|
+
'xact_start' => '2021-07-23T12:55:25.000+00:00'
|
166
|
+
}
|
167
|
+
|
168
|
+
assert_equal expected, record
|
169
|
+
end
|
170
|
+
|
171
|
+
test 'marginalia prepended' do
|
172
|
+
d = create_driver
|
173
|
+
row = {
|
174
|
+
'datid' => 16384,
|
175
|
+
'datname' => 'testuser',
|
176
|
+
'pid' => 376,
|
177
|
+
'usesysid' => 10,
|
178
|
+
'usename' => 'testuser',
|
179
|
+
'application_name' => 'psql',
|
180
|
+
'client_addr' => '172.17.0.1',
|
181
|
+
'client_hostname' => nil,
|
182
|
+
'client_port' => 60182,
|
183
|
+
'xact_start' => Time.parse('2021-07-23 12:55:25 +0000'),
|
184
|
+
'xact_age_s' => 0.001884,
|
185
|
+
'query_start' => Time.parse('2021-07-23 12:55:25 +0000'),
|
186
|
+
'query_age_s' => 0.001894,
|
187
|
+
'state_change' => Time.parse('2021-07-23 12:55:25 +0000'),
|
188
|
+
'state_age_s' => 0.001894,
|
189
|
+
'state' => 'active',
|
190
|
+
'query' => "/*application:web,correlation_id:01F1D2T1SC9DM82A4865ATG1CP,endpoint_id:POST /api/:version/groups/:id/-/packages/mavenpath/:file_name*/ SELECT * FROM users WHERE user_secret = 's3cr3t'"
|
191
|
+
}
|
192
|
+
|
193
|
+
record = d.instance.record_for_row(row)
|
194
|
+
|
195
|
+
expected = {
|
196
|
+
'application' => 'web',
|
197
|
+
'application_name' => 'psql',
|
198
|
+
'client_addr' => '172.17.0.1',
|
199
|
+
'client_hostname' => nil,
|
200
|
+
'client_port' => 60182,
|
201
|
+
'correlation_id' => '01F1D2T1SC9DM82A4865ATG1CP',
|
202
|
+
'datid' => 16384,
|
203
|
+
'datname' => 'testuser',
|
204
|
+
'endpoint_id' => 'POST /api/:version/groups/:id/-/packages/mavenpath/:file_name',
|
205
|
+
'fingerprint' => '5c4a61e156c7d822',
|
206
|
+
'pid' => 376,
|
207
|
+
'query' => 'SELECT * FROM users WHERE user_secret = $1', # Secret removed
|
208
|
+
'query_age_s' => 0.001894,
|
209
|
+
'query_length' => 48,
|
210
|
+
'query_start' => '2021-07-23T12:55:25.000+00:00',
|
211
|
+
'state' => 'active',
|
212
|
+
'state_age_s' => 0.001894,
|
213
|
+
'state_change' => '2021-07-23T12:55:25.000+00:00',
|
214
|
+
'usename' => 'testuser',
|
215
|
+
'usesysid' => 10,
|
216
|
+
'xact_age_s' => 0.001884,
|
217
|
+
'xact_start' => '2021-07-23T12:55:25.000+00:00'
|
218
|
+
}
|
219
|
+
|
220
|
+
assert_equal expected, record
|
221
|
+
end
|
222
|
+
end
|
223
|
+
end
|
@@ -42,13 +42,24 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
|
|
42
42
|
sub_test_case 'execution' do
|
43
43
|
test 'sql' do
|
44
44
|
d = create_driver
|
45
|
-
|
45
|
+
row = {
|
46
|
+
'queryid' => '1234',
|
47
|
+
'query' => 'SELECT * FROM users WHERE user_id = ?',
|
48
|
+
'calls' => 22,
|
49
|
+
'rows' => 333,
|
50
|
+
'total_time' => 44.44
|
51
|
+
}
|
52
|
+
|
53
|
+
record = d.instance.record_for_row(row)
|
46
54
|
|
47
55
|
expected = {
|
48
56
|
'fingerprint' => 'c071dee80d466e7d',
|
49
57
|
'query' => 'SELECT * FROM users WHERE user_id = ?',
|
50
58
|
'query_length' => 37,
|
51
|
-
'queryid' => '1234'
|
59
|
+
'queryid' => '1234',
|
60
|
+
'calls' => 22,
|
61
|
+
'rows' => 333,
|
62
|
+
'total_time_ms' => 44.44
|
52
63
|
}
|
53
64
|
|
54
65
|
assert_equal expected, record
|
@@ -56,9 +67,22 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
|
|
56
67
|
|
57
68
|
test 'nil query' do
|
58
69
|
d = create_driver
|
59
|
-
|
70
|
+
row = {
|
71
|
+
'queryid' => '1234',
|
72
|
+
'query' => nil,
|
73
|
+
'calls' => nil,
|
74
|
+
'rows' => nil,
|
75
|
+
'total_time' => nil
|
76
|
+
}
|
77
|
+
record = d.instance.record_for_row(row)
|
60
78
|
|
61
|
-
expected = {
|
79
|
+
expected = {
|
80
|
+
'query_length' => nil,
|
81
|
+
'queryid' => '1234',
|
82
|
+
'calls' => nil,
|
83
|
+
'rows' => nil,
|
84
|
+
'total_time_ms' => nil
|
85
|
+
}
|
62
86
|
assert_equal expected, record
|
63
87
|
end
|
64
88
|
|
@@ -75,26 +99,48 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
|
|
75
99
|
)
|
76
100
|
SQL
|
77
101
|
|
78
|
-
|
102
|
+
row = {
|
103
|
+
'queryid' => 1234,
|
104
|
+
'query' => ddl_sql,
|
105
|
+
'calls' => 22,
|
106
|
+
'rows' => 333,
|
107
|
+
'total_time' => 44.44
|
108
|
+
}
|
109
|
+
|
110
|
+
record = d.instance.record_for_row(row)
|
79
111
|
|
80
112
|
expected = {
|
81
113
|
'fingerprint' => 'fa9c9d26757c4f9b',
|
82
114
|
'query' => ddl_sql,
|
83
115
|
'query_length' => 287,
|
84
|
-
'queryid' => '1234'
|
116
|
+
'queryid' => '1234',
|
117
|
+
'calls' => 22,
|
118
|
+
'rows' => 333,
|
119
|
+
'total_time_ms' => 44.44
|
85
120
|
}
|
86
121
|
assert_equal expected, record
|
87
122
|
end
|
88
123
|
|
89
124
|
test 'set command' do
|
90
125
|
d = create_driver
|
91
|
-
|
126
|
+
row = {
|
127
|
+
'queryid' => 1234,
|
128
|
+
'query' => "SET TIME ZONE 'PST8PDT'",
|
129
|
+
'calls' => 22,
|
130
|
+
'rows' => 333,
|
131
|
+
'total_time' => 44.44
|
132
|
+
}
|
133
|
+
|
134
|
+
record = d.instance.record_for_row(row)
|
92
135
|
|
93
136
|
expected = {
|
94
137
|
'fingerprint' => '23f8d6eb1d3125c3',
|
95
138
|
'query' => 'SET TIME ZONE $1',
|
96
139
|
'query_length' => 23,
|
97
|
-
'queryid' => '1234'
|
140
|
+
'queryid' => '1234',
|
141
|
+
'calls' => 22,
|
142
|
+
'rows' => 333,
|
143
|
+
'total_time_ms' => 44.44
|
98
144
|
}
|
99
145
|
|
100
146
|
assert_equal expected, record
|
@@ -102,9 +148,17 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
|
|
102
148
|
|
103
149
|
test 'unparseable sql' do
|
104
150
|
d = create_driver
|
105
|
-
record = d.instance.record_for_row({ 'queryid' =>
|
151
|
+
record = d.instance.record_for_row({ 'queryid' => 1234, 'query' => 'SELECT * FROM' })
|
152
|
+
|
153
|
+
expected = {
|
154
|
+
'query_length' => 13,
|
155
|
+
'query_unparseable' => true,
|
156
|
+
'queryid' => '1234',
|
157
|
+
'calls' => nil,
|
158
|
+
'rows' => nil,
|
159
|
+
'total_time_ms' => nil
|
160
|
+
}
|
106
161
|
|
107
|
-
expected = { 'query_length' => 13, 'query_unparseable' => true, 'queryid' => '1234' }
|
108
162
|
assert_equal expected, record
|
109
163
|
end
|
110
164
|
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../helper'
|
4
|
+
|
5
|
+
class Marginalia < Test::Unit::TestCase
|
6
|
+
include Fluent::Plugin::MarginaliaExtractor
|
7
|
+
|
8
|
+
def test_parse(sql, record, key, strip_comment, expected)
|
9
|
+
record[key] = sql
|
10
|
+
parse_marginalia_into_record(record, key, strip_comment)
|
11
|
+
assert_equal(expected, record)
|
12
|
+
end
|
13
|
+
|
14
|
+
test 'no marginalia' do
|
15
|
+
sql = 'SELECT * FROM projects'
|
16
|
+
expected = { 'sql' => 'SELECT * FROM projects' }
|
17
|
+
test_parse(sql, {}, 'sql', true, expected)
|
18
|
+
end
|
19
|
+
|
20
|
+
test 'normal comment appended' do
|
21
|
+
sql = 'SELECT COUNT(*) FROM "projects" /* this is just a comment */'
|
22
|
+
expected = {
|
23
|
+
'sql' => 'SELECT COUNT(*) FROM "projects"'
|
24
|
+
}
|
25
|
+
test_parse(sql, {}, 'sql', true, expected)
|
26
|
+
end
|
27
|
+
|
28
|
+
test 'marginalia appended for sidekiq' do
|
29
|
+
sql = 'SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/'
|
30
|
+
expected = {
|
31
|
+
'application' => 'sidekiq',
|
32
|
+
'correlation_id' => 'd67cae54c169e0cab7d73389e2934f0e',
|
33
|
+
'jid' => '52a1c8a9e4c555ea573f20f0',
|
34
|
+
'job_class' => 'Geo::MetricsUpdateWorker',
|
35
|
+
'sql' => 'SELECT COUNT(*) FROM "projects"'
|
36
|
+
}
|
37
|
+
test_parse(sql, {}, 'sql', true, expected)
|
38
|
+
end
|
39
|
+
|
40
|
+
test 'marginalia appended for web' do
|
41
|
+
sql = 'SELECT COUNT(*) FROM "projects" /*application:web,correlation_id:01F1D2T1SC9DM82A4865ATG1CP,endpoint_id:POST /api/:version/groups/:id/-/packages/mavenpath/:file_name*/'
|
42
|
+
expected = {
|
43
|
+
'application' => 'web',
|
44
|
+
'correlation_id' => '01F1D2T1SC9DM82A4865ATG1CP',
|
45
|
+
'endpoint_id' => 'POST /api/:version/groups/:id/-/packages/mavenpath/:file_name',
|
46
|
+
'sql' => 'SELECT COUNT(*) FROM "projects"'
|
47
|
+
}
|
48
|
+
test_parse(sql, {}, 'sql', true, expected)
|
49
|
+
end
|
50
|
+
|
51
|
+
test 'normal comment prepended' do
|
52
|
+
sql = '/* this is just a comment */ SELECT COUNT(*) FROM "projects"'
|
53
|
+
expected = {
|
54
|
+
"sql" => 'SELECT COUNT(*) FROM "projects"'
|
55
|
+
}
|
56
|
+
test_parse(sql, {}, 'sql', true, expected)
|
57
|
+
end
|
58
|
+
|
59
|
+
test 'marginalia prepended for sidekiq' do
|
60
|
+
sql = '/*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/ SELECT COUNT(*) FROM "projects"'
|
61
|
+
expected = {
|
62
|
+
'application' => 'sidekiq',
|
63
|
+
'correlation_id' => 'd67cae54c169e0cab7d73389e2934f0e',
|
64
|
+
'jid' => '52a1c8a9e4c555ea573f20f0',
|
65
|
+
'job_class' => 'Geo::MetricsUpdateWorker',
|
66
|
+
'sql' => 'SELECT COUNT(*) FROM "projects"'
|
67
|
+
}
|
68
|
+
test_parse(sql, {}, 'sql', true, expected)
|
69
|
+
end
|
70
|
+
|
71
|
+
test 'marginalia prepended for web' do
|
72
|
+
sql = '/*application:web,correlation_id:01F1D2T1SC9DM82A4865ATG1CP,endpoint_id:POST /api/:version/groups/:id/-/packages/mavenpath/:file_name*/ SELECT COUNT(*) FROM "projects"'
|
73
|
+
expected = {
|
74
|
+
'application' => 'web',
|
75
|
+
'correlation_id' => '01F1D2T1SC9DM82A4865ATG1CP',
|
76
|
+
'endpoint_id' => 'POST /api/:version/groups/:id/-/packages/mavenpath/:file_name',
|
77
|
+
'sql' => 'SELECT COUNT(*) FROM "projects"'
|
78
|
+
}
|
79
|
+
test_parse(sql, {}, 'sql', true, expected)
|
80
|
+
end
|
81
|
+
|
82
|
+
test 'marginalia prepended for web, comment_strip disabled' do
|
83
|
+
sql = 'SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/'
|
84
|
+
expected = {
|
85
|
+
'application' => 'sidekiq',
|
86
|
+
'correlation_id' => 'd67cae54c169e0cab7d73389e2934f0e',
|
87
|
+
'jid' => '52a1c8a9e4c555ea573f20f0',
|
88
|
+
'job_class' => 'Geo::MetricsUpdateWorker',
|
89
|
+
'sql' => sql
|
90
|
+
}
|
91
|
+
test_parse(sql, { 'sql' => sql }, 'sql', false, expected)
|
92
|
+
end
|
93
|
+
|
94
|
+
test 'avoid clash' do
|
95
|
+
sql = '/*clash_key:bbb*/ SELECT COUNT(*) FROM "projects"'
|
96
|
+
expected = {
|
97
|
+
'clash_key' => 'aaa',
|
98
|
+
'sql_clash_key' => 'bbb',
|
99
|
+
'sql' => 'SELECT COUNT(*) FROM "projects"'
|
100
|
+
}
|
101
|
+
test_parse(sql, { 'clash_key' => 'aaa' }, 'sql', true, expected)
|
102
|
+
end
|
103
|
+
end
|