fluent-plugin-postgresql-csvlog 0.3.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.gitlab-ci.yml +27 -3
- data/README.md +17 -0
- data/docker-compose.yml +26 -8
- data/example-fluentd.conf +30 -1
- data/fluent-plugin-postgresql-csvlog.gemspec +1 -1
- data/lib/fluent/plugin/filter_marginalia.rb +3 -67
- data/lib/fluent/plugin/in_pg_stat_activity.rb +118 -0
- data/lib/fluent/plugin/in_pg_stat_statements.rb +59 -77
- data/lib/fluent/plugin/marginalia_extractor.rb +73 -0
- data/lib/fluent/plugin/polling_pg_input_plugin.rb +100 -0
- data/test/helper.rb +2 -0
- data/test/plugin/itest_in_pg_stat_activity.rb +115 -0
- data/test/plugin/itest_in_pg_stat_statements.rb +3 -3
- data/test/plugin/test_in_pg_stat_activity.rb +223 -0
- data/test/plugin/test_in_pg_stat_statements.rb +64 -10
- data/test/plugin/test_marginalia_extractor.rb +103 -0
- data/test/verify-docker-compose.sh +32 -0
- metadata +18 -6
@@ -0,0 +1,100 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'fluent/plugin/input'
|
4
|
+
require 'pg'
|
5
|
+
require 'pg_query'
|
6
|
+
|
7
|
+
module Fluent::Plugin
|
8
|
+
# PollingPostgresInputPlugin is intended to be used as an base class
|
9
|
+
# for input plugins that poll postgres.
|
10
|
+
#
|
11
|
+
# Child classes should implement the `on_poll` method
|
12
|
+
class PollingPostgresInputPlugin < Input
|
13
|
+
desc 'PostgreSQL host'
|
14
|
+
config_param :host, :string
|
15
|
+
|
16
|
+
desc 'RDBMS port (default: 5432)'
|
17
|
+
config_param :port, :integer, default: 5432
|
18
|
+
|
19
|
+
desc 'login user name'
|
20
|
+
config_param :username, :string, default: nil
|
21
|
+
|
22
|
+
desc 'postgres db'
|
23
|
+
config_param :dbname, :string, default: nil
|
24
|
+
|
25
|
+
desc 'login password'
|
26
|
+
config_param :password, :string, default: nil, secret: true
|
27
|
+
|
28
|
+
# See https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE
|
29
|
+
# for options
|
30
|
+
desc 'postgres sslmode'
|
31
|
+
config_param :sslmode, :string, default: 'prefer'
|
32
|
+
|
33
|
+
desc 'tag'
|
34
|
+
config_param :tag, :string, default: nil
|
35
|
+
|
36
|
+
desc 'interval in second to run query'
|
37
|
+
config_param :interval, :time, default: 300
|
38
|
+
|
39
|
+
def start
|
40
|
+
@stop_flag = false
|
41
|
+
@thread = Thread.new(&method(:thread_main))
|
42
|
+
end
|
43
|
+
|
44
|
+
# Fluentd shutdown method, called to terminate and cleanup plugin
|
45
|
+
def shutdown
|
46
|
+
@stop_flag = true
|
47
|
+
|
48
|
+
# Interrupt thread and wait for it to finish
|
49
|
+
Thread.new { @thread.run } if @thread
|
50
|
+
@thread.join
|
51
|
+
end
|
52
|
+
|
53
|
+
# Main polling loop on thread
|
54
|
+
def thread_main
|
55
|
+
until @stop_flag
|
56
|
+
sleep @interval
|
57
|
+
break if @stop_flag
|
58
|
+
|
59
|
+
begin
|
60
|
+
on_poll
|
61
|
+
rescue StandardError => e
|
62
|
+
log.error 'unexpected error', error: e.message, error_class: e.class
|
63
|
+
log.error_backtrace
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
protected
|
69
|
+
|
70
|
+
# Child-classes should implement this method
|
71
|
+
def on_poll
|
72
|
+
raise 'on_poll must be implemented by descendents of PollingPostgresInputPlugin'
|
73
|
+
end
|
74
|
+
|
75
|
+
# Since this query is very infrequent, and it may be communicating directly
|
76
|
+
# with postgres without pgbouncer, don't use a persistent connection and
|
77
|
+
# ensure that it is properly closed
|
78
|
+
def with_connection(&block)
|
79
|
+
conn = PG.connect(
|
80
|
+
host: @host,
|
81
|
+
dbname: @dbname,
|
82
|
+
sslmode: @sslmode,
|
83
|
+
user: @username,
|
84
|
+
password: @password
|
85
|
+
)
|
86
|
+
|
87
|
+
map = PG::BasicTypeMapForResults.new(conn)
|
88
|
+
map.default_type_map = PG::TypeMapAllStrings.new
|
89
|
+
|
90
|
+
conn.type_map_for_results = map
|
91
|
+
|
92
|
+
begin
|
93
|
+
block.call(conn)
|
94
|
+
ensure
|
95
|
+
# Always close the connection
|
96
|
+
conn.finish
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
data/test/helper.rb
CHANGED
@@ -16,4 +16,6 @@ Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
|
16
16
|
require 'fluent/plugin/filter_postgresql_slowlog'
|
17
17
|
require 'fluent/plugin/filter_postgresql_redactor'
|
18
18
|
require 'fluent/plugin/filter_marginalia'
|
19
|
+
require 'fluent/plugin/marginalia_extractor'
|
19
20
|
require 'fluent/plugin/in_pg_stat_statements'
|
21
|
+
require 'fluent/plugin/in_pg_stat_activity'
|
@@ -0,0 +1,115 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../helper'
|
4
|
+
|
5
|
+
class PgStatActivityInputIntegrationTest < Test::Unit::TestCase
|
6
|
+
# The defaults values work with the configuration in .gitlab-ci.yml on the postgres service
|
7
|
+
# Override with env vars for local development
|
8
|
+
HOST = ENV.fetch('PG_TEST_HOST', 'postgres')
|
9
|
+
USERNAME = ENV.fetch('PG_TEST_USER', 'testuser')
|
10
|
+
PASSWORD = ENV.fetch('PG_TEST_PASSWORD', 'testpass')
|
11
|
+
|
12
|
+
def setup
|
13
|
+
Fluent::Test.setup
|
14
|
+
|
15
|
+
@conn = PG.connect(
|
16
|
+
host: HOST,
|
17
|
+
user: USERNAME,
|
18
|
+
password: PASSWORD
|
19
|
+
)
|
20
|
+
end
|
21
|
+
|
22
|
+
def teardown
|
23
|
+
@conn&.finish
|
24
|
+
end
|
25
|
+
|
26
|
+
VALID_CONFIG = %(
|
27
|
+
tag postgres.pg_stat_statements
|
28
|
+
host #{HOST}
|
29
|
+
username #{USERNAME}
|
30
|
+
password #{PASSWORD}
|
31
|
+
interval 1
|
32
|
+
)
|
33
|
+
|
34
|
+
INVALID_CONFIG = %(
|
35
|
+
host 'invalid_host.dne'
|
36
|
+
port 1234
|
37
|
+
username #{USERNAME}
|
38
|
+
password #{PASSWORD}
|
39
|
+
interval 1
|
40
|
+
)
|
41
|
+
|
42
|
+
def create_driver(config)
|
43
|
+
Fluent::Test::InputTestDriver.new(Fluent::Plugin::PgStatActivityInput).configure(config)
|
44
|
+
end
|
45
|
+
|
46
|
+
sub_test_case 'configuration' do
|
47
|
+
test 'connects' do
|
48
|
+
d = create_driver(VALID_CONFIG)
|
49
|
+
|
50
|
+
emits = []
|
51
|
+
# wait 50 * 0.05, "see fluentd/lib/fluent/test/base.rb:79 num_waits.times { sleep 0.05 }
|
52
|
+
d.run(num_waits = 50) do
|
53
|
+
emits = d.emits
|
54
|
+
end
|
55
|
+
|
56
|
+
assert_false emits.empty?
|
57
|
+
end
|
58
|
+
|
59
|
+
# Why do we have this test? If postgres is still starting up, we don't want to cause the
|
60
|
+
# the fluentd configuration to fail. We would rather retry until we get a connection
|
61
|
+
test 'connects for an invalid config' do
|
62
|
+
d = create_driver(INVALID_CONFIG)
|
63
|
+
|
64
|
+
emits = []
|
65
|
+
# wait 50 * 0.05, "see fluentd/lib/fluent/test/base.rb:79 num_waits.times { sleep 0.05 }
|
66
|
+
d.run(num_waits = 50) do
|
67
|
+
emits = d.emits
|
68
|
+
end
|
69
|
+
|
70
|
+
assert_true emits.empty?
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
sub_test_case 'execution' do
|
75
|
+
test 'connects' do
|
76
|
+
d = create_driver(VALID_CONFIG)
|
77
|
+
|
78
|
+
emits = []
|
79
|
+
# wait 50 * 0.05, "see fluentd/lib/fluent/test/base.rb:79 num_waits.times { sleep 0.05 }
|
80
|
+
d.run(num_waits = 50) do
|
81
|
+
emits = d.emits
|
82
|
+
end
|
83
|
+
|
84
|
+
first_with_query = emits.find do |event|
|
85
|
+
record = event[2]
|
86
|
+
|
87
|
+
record['usename'] == USERNAME &&
|
88
|
+
!record['datid'].nil? &&
|
89
|
+
!record['query'].nil? &&
|
90
|
+
record['state'] == 'active'
|
91
|
+
end
|
92
|
+
|
93
|
+
assert_false first_with_query.nil?
|
94
|
+
record = first_with_query[2]
|
95
|
+
|
96
|
+
assert_false record['datname'].nil?
|
97
|
+
assert_false record['pid'].nil?
|
98
|
+
assert_false record['usesysid'].nil?
|
99
|
+
assert_false record['application_name'].nil?
|
100
|
+
assert_false record['client_addr'].nil?
|
101
|
+
assert_false record['client_port'].nil?
|
102
|
+
assert_false record['xact_start'].nil?
|
103
|
+
assert_false record['xact_age_s'].nil?
|
104
|
+
assert_false record['xact_start'].nil?
|
105
|
+
assert_false record['xact_age_s'].nil?
|
106
|
+
assert_false record['query_start'].nil?
|
107
|
+
assert_false record['query_age_s'].nil?
|
108
|
+
assert_false record['state_change'].nil?
|
109
|
+
assert_false record['state_age_s'].nil?
|
110
|
+
assert_false record['query_length'].nil?
|
111
|
+
assert_false record['query'].nil?
|
112
|
+
assert_false record['fingerprint'].nil?
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -98,8 +98,7 @@ class PgStatStatementsInputIntegrationTest < Test::Unit::TestCase
|
|
98
98
|
expected_record = {
|
99
99
|
'fingerprint' => '8a6e9896bd9048a2',
|
100
100
|
'query' => 'SELECT * FROM pg_stat_statements ORDER BY queryid LIMIT $1',
|
101
|
-
'query_length' => 58
|
102
|
-
'queryid' => 3_239_318_621_761_098_074
|
101
|
+
'query_length' => 58
|
103
102
|
}
|
104
103
|
known_statement_event = emits.find do |event|
|
105
104
|
record = event[2]
|
@@ -114,7 +113,8 @@ class PgStatStatementsInputIntegrationTest < Test::Unit::TestCase
|
|
114
113
|
assert_equal 'postgres.pg_stat_statements', tag
|
115
114
|
assert_equal expected_record['fingerprint'], record['fingerprint']
|
116
115
|
assert_equal expected_record['query_length'], record['query_length']
|
117
|
-
assert_true
|
116
|
+
assert_true record.include? 'queryid'
|
117
|
+
assert_true record['queryid'].is_a? String
|
118
118
|
end
|
119
119
|
end
|
120
120
|
end
|
@@ -0,0 +1,223 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../helper'
|
4
|
+
require 'date'
|
5
|
+
|
6
|
+
class PgStatActivityInputTest < Test::Unit::TestCase
|
7
|
+
def setup
|
8
|
+
Fluent::Test.setup
|
9
|
+
end
|
10
|
+
|
11
|
+
CONFIG = %(
|
12
|
+
tag postgres.pg_stat_activity
|
13
|
+
host localhost
|
14
|
+
port 1234
|
15
|
+
dbname gitlab
|
16
|
+
sslmode require
|
17
|
+
username moo
|
18
|
+
password secret
|
19
|
+
interval 600
|
20
|
+
fingerprint_key fingerprint
|
21
|
+
)
|
22
|
+
|
23
|
+
def create_driver
|
24
|
+
Fluent::Test::InputTestDriver.new(Fluent::Plugin::PgStatActivityInput).configure(CONFIG)
|
25
|
+
end
|
26
|
+
|
27
|
+
sub_test_case 'configuration' do
|
28
|
+
test 'basic configuration' do
|
29
|
+
d = create_driver
|
30
|
+
|
31
|
+
assert_equal 'postgres.pg_stat_activity', d.instance.tag
|
32
|
+
assert_equal 'localhost', d.instance.host
|
33
|
+
assert_equal 1234, d.instance.port
|
34
|
+
assert_equal 'gitlab', d.instance.dbname
|
35
|
+
assert_equal 'require', d.instance.sslmode
|
36
|
+
assert_equal 'moo', d.instance.username
|
37
|
+
assert_equal 'secret', d.instance.password
|
38
|
+
assert_equal 600, d.instance.interval
|
39
|
+
assert_equal 'fingerprint', d.instance.fingerprint_key
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
sub_test_case 'execution' do
|
44
|
+
test 'sql' do
|
45
|
+
d = create_driver
|
46
|
+
row = {
|
47
|
+
'datid' => 16384,
|
48
|
+
'datname' => 'testuser',
|
49
|
+
'pid' => 376,
|
50
|
+
'usesysid' => 10,
|
51
|
+
'usename' => 'testuser',
|
52
|
+
'application_name' => 'psql',
|
53
|
+
'client_addr' => '172.17.0.1',
|
54
|
+
'client_hostname' => nil,
|
55
|
+
'client_port' => 60182,
|
56
|
+
'xact_start' => Time.parse('2021-07-23 12:55:25 +0000'),
|
57
|
+
'xact_age_s' => 0.001884,
|
58
|
+
'query_start' => Time.parse('2021-07-23 12:55:25 +0000'),
|
59
|
+
'query_age_s' => 0.001894,
|
60
|
+
'state_change' => Time.parse('2021-07-23 12:55:25 +0000'),
|
61
|
+
'state_age_s' => 0.001894,
|
62
|
+
'state' => 'active',
|
63
|
+
'query' => "SELECT * FROM users WHERE user_secret = 's3cr3t'"
|
64
|
+
}
|
65
|
+
|
66
|
+
record = d.instance.record_for_row(row)
|
67
|
+
|
68
|
+
expected = {
|
69
|
+
'application_name' => 'psql',
|
70
|
+
'client_addr' => '172.17.0.1',
|
71
|
+
'client_hostname' => nil,
|
72
|
+
'client_port' => 60182,
|
73
|
+
'datid' => 16384,
|
74
|
+
'datname' => 'testuser',
|
75
|
+
'fingerprint' => '5c4a61e156c7d822',
|
76
|
+
'pid' => 376,
|
77
|
+
'query' => 'SELECT * FROM users WHERE user_secret = $1', # NOTE: secret removed
|
78
|
+
'query_age_s' => 0.001894,
|
79
|
+
'query_length' => 48,
|
80
|
+
'query_start' => '2021-07-23T12:55:25.000+00:00',
|
81
|
+
'state' => 'active',
|
82
|
+
'state_age_s' => 0.001894,
|
83
|
+
'state_change' => '2021-07-23T12:55:25.000+00:00',
|
84
|
+
'usename' => 'testuser',
|
85
|
+
'usesysid' => 10,
|
86
|
+
'xact_age_s' => 0.001884,
|
87
|
+
'xact_start' => '2021-07-23T12:55:25.000+00:00'
|
88
|
+
}
|
89
|
+
|
90
|
+
assert_equal expected, record
|
91
|
+
end
|
92
|
+
|
93
|
+
# This test mostly checks that the code is nil safe
|
94
|
+
test 'nil query' do
|
95
|
+
d = create_driver
|
96
|
+
record = d.instance.record_for_row({})
|
97
|
+
|
98
|
+
expected = {
|
99
|
+
'application_name' => nil,
|
100
|
+
'client_addr' => nil,
|
101
|
+
'client_hostname' => nil,
|
102
|
+
'client_port' => nil,
|
103
|
+
'datid' => nil,
|
104
|
+
'datname' => nil,
|
105
|
+
'pid' => nil,
|
106
|
+
'query' => nil,
|
107
|
+
'query_age_s' => nil,
|
108
|
+
'query_length' => nil,
|
109
|
+
'query_start' => nil,
|
110
|
+
'state' => nil,
|
111
|
+
'state_age_s' => nil,
|
112
|
+
'state_change' => nil,
|
113
|
+
'usename' => nil,
|
114
|
+
'usesysid' => nil,
|
115
|
+
'xact_age_s' => nil,
|
116
|
+
'xact_start' => nil
|
117
|
+
}
|
118
|
+
|
119
|
+
assert_equal expected, record
|
120
|
+
end
|
121
|
+
|
122
|
+
test 'unparseable sql' do
|
123
|
+
d = create_driver
|
124
|
+
row = {
|
125
|
+
'datid' => 16384,
|
126
|
+
'datname' => 'testuser',
|
127
|
+
'pid' => 376,
|
128
|
+
'usesysid' => 10,
|
129
|
+
'usename' => 'testuser',
|
130
|
+
'application_name' => 'psql',
|
131
|
+
'client_addr' => '172.17.0.1',
|
132
|
+
'client_hostname' => nil,
|
133
|
+
'client_port' => 60182,
|
134
|
+
'xact_start' => Time.parse('2021-07-23 12:55:25 +0000'),
|
135
|
+
'xact_age_s' => 0.001884,
|
136
|
+
'query_start' => Time.parse('2021-07-23 12:55:25 +0000'),
|
137
|
+
'query_age_s' => 0.001894,
|
138
|
+
'state_change' => Time.parse('2021-07-23 12:55:25 +0000'),
|
139
|
+
'state_age_s' => 0.001894,
|
140
|
+
'state' => 'active',
|
141
|
+
'query' => "SELECT * FROM users WHERE user_se="
|
142
|
+
}
|
143
|
+
|
144
|
+
record = d.instance.record_for_row(row)
|
145
|
+
|
146
|
+
expected = {
|
147
|
+
'application_name' => 'psql',
|
148
|
+
'client_addr' => '172.17.0.1',
|
149
|
+
'client_hostname' => nil,
|
150
|
+
'client_port' => 60182,
|
151
|
+
'datid' => 16384,
|
152
|
+
'datname' => 'testuser',
|
153
|
+
'pid' => 376,
|
154
|
+
'query' => nil,
|
155
|
+
'query_age_s' => 0.001894,
|
156
|
+
'query_length' => 34,
|
157
|
+
'query_start' => '2021-07-23T12:55:25.000+00:00',
|
158
|
+
'query_unparseable' => true,
|
159
|
+
'state' => 'active',
|
160
|
+
'state_age_s' => 0.001894,
|
161
|
+
'state_change' => '2021-07-23T12:55:25.000+00:00',
|
162
|
+
'usename' => 'testuser',
|
163
|
+
'usesysid' => 10,
|
164
|
+
'xact_age_s' => 0.001884,
|
165
|
+
'xact_start' => '2021-07-23T12:55:25.000+00:00'
|
166
|
+
}
|
167
|
+
|
168
|
+
assert_equal expected, record
|
169
|
+
end
|
170
|
+
|
171
|
+
test 'marginalia prepended' do
|
172
|
+
d = create_driver
|
173
|
+
row = {
|
174
|
+
'datid' => 16384,
|
175
|
+
'datname' => 'testuser',
|
176
|
+
'pid' => 376,
|
177
|
+
'usesysid' => 10,
|
178
|
+
'usename' => 'testuser',
|
179
|
+
'application_name' => 'psql',
|
180
|
+
'client_addr' => '172.17.0.1',
|
181
|
+
'client_hostname' => nil,
|
182
|
+
'client_port' => 60182,
|
183
|
+
'xact_start' => Time.parse('2021-07-23 12:55:25 +0000'),
|
184
|
+
'xact_age_s' => 0.001884,
|
185
|
+
'query_start' => Time.parse('2021-07-23 12:55:25 +0000'),
|
186
|
+
'query_age_s' => 0.001894,
|
187
|
+
'state_change' => Time.parse('2021-07-23 12:55:25 +0000'),
|
188
|
+
'state_age_s' => 0.001894,
|
189
|
+
'state' => 'active',
|
190
|
+
'query' => "/*application:web,correlation_id:01F1D2T1SC9DM82A4865ATG1CP,endpoint_id:POST /api/:version/groups/:id/-/packages/mavenpath/:file_name*/ SELECT * FROM users WHERE user_secret = 's3cr3t'"
|
191
|
+
}
|
192
|
+
|
193
|
+
record = d.instance.record_for_row(row)
|
194
|
+
|
195
|
+
expected = {
|
196
|
+
'application' => 'web',
|
197
|
+
'application_name' => 'psql',
|
198
|
+
'client_addr' => '172.17.0.1',
|
199
|
+
'client_hostname' => nil,
|
200
|
+
'client_port' => 60182,
|
201
|
+
'correlation_id' => '01F1D2T1SC9DM82A4865ATG1CP',
|
202
|
+
'datid' => 16384,
|
203
|
+
'datname' => 'testuser',
|
204
|
+
'endpoint_id' => 'POST /api/:version/groups/:id/-/packages/mavenpath/:file_name',
|
205
|
+
'fingerprint' => '5c4a61e156c7d822',
|
206
|
+
'pid' => 376,
|
207
|
+
'query' => 'SELECT * FROM users WHERE user_secret = $1', # Secret removed
|
208
|
+
'query_age_s' => 0.001894,
|
209
|
+
'query_length' => 48,
|
210
|
+
'query_start' => '2021-07-23T12:55:25.000+00:00',
|
211
|
+
'state' => 'active',
|
212
|
+
'state_age_s' => 0.001894,
|
213
|
+
'state_change' => '2021-07-23T12:55:25.000+00:00',
|
214
|
+
'usename' => 'testuser',
|
215
|
+
'usesysid' => 10,
|
216
|
+
'xact_age_s' => 0.001884,
|
217
|
+
'xact_start' => '2021-07-23T12:55:25.000+00:00'
|
218
|
+
}
|
219
|
+
|
220
|
+
assert_equal expected, record
|
221
|
+
end
|
222
|
+
end
|
223
|
+
end
|
@@ -42,13 +42,24 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
|
|
42
42
|
sub_test_case 'execution' do
|
43
43
|
test 'sql' do
|
44
44
|
d = create_driver
|
45
|
-
|
45
|
+
row = {
|
46
|
+
'queryid' => '1234',
|
47
|
+
'query' => 'SELECT * FROM users WHERE user_id = ?',
|
48
|
+
'calls' => 22,
|
49
|
+
'rows' => 333,
|
50
|
+
'total_time' => 44.44
|
51
|
+
}
|
52
|
+
|
53
|
+
record = d.instance.record_for_row(row)
|
46
54
|
|
47
55
|
expected = {
|
48
56
|
'fingerprint' => 'c071dee80d466e7d',
|
49
57
|
'query' => 'SELECT * FROM users WHERE user_id = ?',
|
50
58
|
'query_length' => 37,
|
51
|
-
'queryid' => '1234'
|
59
|
+
'queryid' => '1234',
|
60
|
+
'calls' => 22,
|
61
|
+
'rows' => 333,
|
62
|
+
'total_time_ms' => 44.44
|
52
63
|
}
|
53
64
|
|
54
65
|
assert_equal expected, record
|
@@ -56,9 +67,22 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
|
|
56
67
|
|
57
68
|
test 'nil query' do
|
58
69
|
d = create_driver
|
59
|
-
|
70
|
+
row = {
|
71
|
+
'queryid' => '1234',
|
72
|
+
'query' => nil,
|
73
|
+
'calls' => nil,
|
74
|
+
'rows' => nil,
|
75
|
+
'total_time' => nil
|
76
|
+
}
|
77
|
+
record = d.instance.record_for_row(row)
|
60
78
|
|
61
|
-
expected = {
|
79
|
+
expected = {
|
80
|
+
'query_length' => nil,
|
81
|
+
'queryid' => '1234',
|
82
|
+
'calls' => nil,
|
83
|
+
'rows' => nil,
|
84
|
+
'total_time_ms' => nil
|
85
|
+
}
|
62
86
|
assert_equal expected, record
|
63
87
|
end
|
64
88
|
|
@@ -75,26 +99,48 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
|
|
75
99
|
)
|
76
100
|
SQL
|
77
101
|
|
78
|
-
|
102
|
+
row = {
|
103
|
+
'queryid' => 1234,
|
104
|
+
'query' => ddl_sql,
|
105
|
+
'calls' => 22,
|
106
|
+
'rows' => 333,
|
107
|
+
'total_time' => 44.44
|
108
|
+
}
|
109
|
+
|
110
|
+
record = d.instance.record_for_row(row)
|
79
111
|
|
80
112
|
expected = {
|
81
113
|
'fingerprint' => 'fa9c9d26757c4f9b',
|
82
114
|
'query' => ddl_sql,
|
83
115
|
'query_length' => 287,
|
84
|
-
'queryid' => '1234'
|
116
|
+
'queryid' => '1234',
|
117
|
+
'calls' => 22,
|
118
|
+
'rows' => 333,
|
119
|
+
'total_time_ms' => 44.44
|
85
120
|
}
|
86
121
|
assert_equal expected, record
|
87
122
|
end
|
88
123
|
|
89
124
|
test 'set command' do
|
90
125
|
d = create_driver
|
91
|
-
|
126
|
+
row = {
|
127
|
+
'queryid' => 1234,
|
128
|
+
'query' => "SET TIME ZONE 'PST8PDT'",
|
129
|
+
'calls' => 22,
|
130
|
+
'rows' => 333,
|
131
|
+
'total_time' => 44.44
|
132
|
+
}
|
133
|
+
|
134
|
+
record = d.instance.record_for_row(row)
|
92
135
|
|
93
136
|
expected = {
|
94
137
|
'fingerprint' => '23f8d6eb1d3125c3',
|
95
138
|
'query' => 'SET TIME ZONE $1',
|
96
139
|
'query_length' => 23,
|
97
|
-
'queryid' => '1234'
|
140
|
+
'queryid' => '1234',
|
141
|
+
'calls' => 22,
|
142
|
+
'rows' => 333,
|
143
|
+
'total_time_ms' => 44.44
|
98
144
|
}
|
99
145
|
|
100
146
|
assert_equal expected, record
|
@@ -102,9 +148,17 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
|
|
102
148
|
|
103
149
|
test 'unparseable sql' do
|
104
150
|
d = create_driver
|
105
|
-
record = d.instance.record_for_row({ 'queryid' =>
|
151
|
+
record = d.instance.record_for_row({ 'queryid' => 1234, 'query' => 'SELECT * FROM' })
|
152
|
+
|
153
|
+
expected = {
|
154
|
+
'query_length' => 13,
|
155
|
+
'query_unparseable' => true,
|
156
|
+
'queryid' => '1234',
|
157
|
+
'calls' => nil,
|
158
|
+
'rows' => nil,
|
159
|
+
'total_time_ms' => nil
|
160
|
+
}
|
106
161
|
|
107
|
-
expected = { 'query_length' => 13, 'query_unparseable' => true, 'queryid' => '1234' }
|
108
162
|
assert_equal expected, record
|
109
163
|
end
|
110
164
|
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../helper'
|
4
|
+
|
5
|
+
class Marginalia < Test::Unit::TestCase
|
6
|
+
include Fluent::Plugin::MarginaliaExtractor
|
7
|
+
|
8
|
+
def test_parse(sql, record, key, strip_comment, expected)
|
9
|
+
record[key] = sql
|
10
|
+
parse_marginalia_into_record(record, key, strip_comment)
|
11
|
+
assert_equal(expected, record)
|
12
|
+
end
|
13
|
+
|
14
|
+
test 'no marginalia' do
|
15
|
+
sql = 'SELECT * FROM projects'
|
16
|
+
expected = { 'sql' => 'SELECT * FROM projects' }
|
17
|
+
test_parse(sql, {}, 'sql', true, expected)
|
18
|
+
end
|
19
|
+
|
20
|
+
test 'normal comment appended' do
|
21
|
+
sql = 'SELECT COUNT(*) FROM "projects" /* this is just a comment */'
|
22
|
+
expected = {
|
23
|
+
'sql' => 'SELECT COUNT(*) FROM "projects"'
|
24
|
+
}
|
25
|
+
test_parse(sql, {}, 'sql', true, expected)
|
26
|
+
end
|
27
|
+
|
28
|
+
test 'marginalia appended for sidekiq' do
|
29
|
+
sql = 'SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/'
|
30
|
+
expected = {
|
31
|
+
'application' => 'sidekiq',
|
32
|
+
'correlation_id' => 'd67cae54c169e0cab7d73389e2934f0e',
|
33
|
+
'jid' => '52a1c8a9e4c555ea573f20f0',
|
34
|
+
'job_class' => 'Geo::MetricsUpdateWorker',
|
35
|
+
'sql' => 'SELECT COUNT(*) FROM "projects"'
|
36
|
+
}
|
37
|
+
test_parse(sql, {}, 'sql', true, expected)
|
38
|
+
end
|
39
|
+
|
40
|
+
test 'marginalia appended for web' do
|
41
|
+
sql = 'SELECT COUNT(*) FROM "projects" /*application:web,correlation_id:01F1D2T1SC9DM82A4865ATG1CP,endpoint_id:POST /api/:version/groups/:id/-/packages/mavenpath/:file_name*/'
|
42
|
+
expected = {
|
43
|
+
'application' => 'web',
|
44
|
+
'correlation_id' => '01F1D2T1SC9DM82A4865ATG1CP',
|
45
|
+
'endpoint_id' => 'POST /api/:version/groups/:id/-/packages/mavenpath/:file_name',
|
46
|
+
'sql' => 'SELECT COUNT(*) FROM "projects"'
|
47
|
+
}
|
48
|
+
test_parse(sql, {}, 'sql', true, expected)
|
49
|
+
end
|
50
|
+
|
51
|
+
test 'normal comment prepended' do
|
52
|
+
sql = '/* this is just a comment */ SELECT COUNT(*) FROM "projects"'
|
53
|
+
expected = {
|
54
|
+
"sql" => 'SELECT COUNT(*) FROM "projects"'
|
55
|
+
}
|
56
|
+
test_parse(sql, {}, 'sql', true, expected)
|
57
|
+
end
|
58
|
+
|
59
|
+
test 'marginalia prepended for sidekiq' do
|
60
|
+
sql = '/*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/ SELECT COUNT(*) FROM "projects"'
|
61
|
+
expected = {
|
62
|
+
'application' => 'sidekiq',
|
63
|
+
'correlation_id' => 'd67cae54c169e0cab7d73389e2934f0e',
|
64
|
+
'jid' => '52a1c8a9e4c555ea573f20f0',
|
65
|
+
'job_class' => 'Geo::MetricsUpdateWorker',
|
66
|
+
'sql' => 'SELECT COUNT(*) FROM "projects"'
|
67
|
+
}
|
68
|
+
test_parse(sql, {}, 'sql', true, expected)
|
69
|
+
end
|
70
|
+
|
71
|
+
test 'marginalia prepended for web' do
|
72
|
+
sql = '/*application:web,correlation_id:01F1D2T1SC9DM82A4865ATG1CP,endpoint_id:POST /api/:version/groups/:id/-/packages/mavenpath/:file_name*/ SELECT COUNT(*) FROM "projects"'
|
73
|
+
expected = {
|
74
|
+
'application' => 'web',
|
75
|
+
'correlation_id' => '01F1D2T1SC9DM82A4865ATG1CP',
|
76
|
+
'endpoint_id' => 'POST /api/:version/groups/:id/-/packages/mavenpath/:file_name',
|
77
|
+
'sql' => 'SELECT COUNT(*) FROM "projects"'
|
78
|
+
}
|
79
|
+
test_parse(sql, {}, 'sql', true, expected)
|
80
|
+
end
|
81
|
+
|
82
|
+
test 'marginalia prepended for web, comment_strip disabled' do
|
83
|
+
sql = 'SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/'
|
84
|
+
expected = {
|
85
|
+
'application' => 'sidekiq',
|
86
|
+
'correlation_id' => 'd67cae54c169e0cab7d73389e2934f0e',
|
87
|
+
'jid' => '52a1c8a9e4c555ea573f20f0',
|
88
|
+
'job_class' => 'Geo::MetricsUpdateWorker',
|
89
|
+
'sql' => sql
|
90
|
+
}
|
91
|
+
test_parse(sql, { 'sql' => sql }, 'sql', false, expected)
|
92
|
+
end
|
93
|
+
|
94
|
+
test 'avoid clash' do
|
95
|
+
sql = '/*clash_key:bbb*/ SELECT COUNT(*) FROM "projects"'
|
96
|
+
expected = {
|
97
|
+
'clash_key' => 'aaa',
|
98
|
+
'sql_clash_key' => 'bbb',
|
99
|
+
'sql' => 'SELECT COUNT(*) FROM "projects"'
|
100
|
+
}
|
101
|
+
test_parse(sql, { 'clash_key' => 'aaa' }, 'sql', true, expected)
|
102
|
+
end
|
103
|
+
end
|