fluent-plugin-postgresql-csvlog 0.6.0 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +14 -3
- data/README.md +2 -1
- data/docker-compose.yml +1 -1
- data/fluent-plugin-postgresql-csvlog.gemspec +1 -1
- data/lib/fluent/plugin/in_pg_stat_statements.rb +53 -2
- data/lib/fluent/plugin/polling_pg_input_plugin.rb +1 -1
- data/test/plugin/test_in_pg_stat_statements.rb +63 -9
- data/test/verify-docker-compose.sh +10 -6
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 38090772bf1e0302ecd53d759d8da4ffb2df0cdf0a6f001ec0e0d12390472276
|
4
|
+
data.tar.gz: 22bc8479b81f0b0fd7615cfb6cc1b491b317fc74d612f21373243a516e671be4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c3ecfef7d3290ab00a71613c7914c0aae1d05501d37574614d9d6f4a2f5de9548474ecb7b6fdad4ef778c4ea48510a1e31a89294e317ecf7ab6004e094579cc9
|
7
|
+
data.tar.gz: 2b91a1704d53f07ab5139bc2d623c0c17afde53b416ef5f018dd7e6d345ce8a8da1d0aa8964bcef638b8e6ab351b2d7c16b83be73cbdcfba560dda69962bdd17
|
data/.gitlab-ci.yml
CHANGED
@@ -10,10 +10,9 @@ test:
|
|
10
10
|
paths:
|
11
11
|
- vendor/ruby
|
12
12
|
|
13
|
-
|
14
|
-
itest:
|
13
|
+
.iteration_test:
|
15
14
|
services:
|
16
|
-
- name: postgres
|
15
|
+
- name: postgres:$POSTGRES_SERVER_VERSION
|
17
16
|
alias: postgres
|
18
17
|
command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all"]
|
19
18
|
variables:
|
@@ -28,6 +27,18 @@ itest:
|
|
28
27
|
paths:
|
29
28
|
- vendor/ruby
|
30
29
|
|
30
|
+
# integration tests for postgres 12
|
31
|
+
itest_pg12:
|
32
|
+
extends: .iteration_test
|
33
|
+
variables:
|
34
|
+
POSTGRES_SERVER_VERSION: 12
|
35
|
+
|
36
|
+
# integration tests for postgres 13
|
37
|
+
itest_pg13:
|
38
|
+
extends: .iteration_test
|
39
|
+
variables:
|
40
|
+
POSTGRES_SERVER_VERSION: 13
|
41
|
+
|
31
42
|
end_to_end_verification_test:
|
32
43
|
image: docker:19.03.12
|
33
44
|
services:
|
data/README.md
CHANGED
@@ -80,7 +80,8 @@ ingest and parse PostgreSQL CSV logs:
|
|
80
80
|
To develop and debug locally, there is a `Dockerfile` and `docker-compose.yml` that will setup a local environment,
|
81
81
|
complete with Postgres, suitable for testing purposes.
|
82
82
|
|
83
|
-
1. `docker compose
|
83
|
+
1. `docker compose build` - build the current configuration
|
84
|
+
1. `docker compose run --rm verifier` - test the current configuration
|
84
85
|
1. `docker compose up`
|
85
86
|
|
86
87
|
### Releasing a new version
|
data/docker-compose.yml
CHANGED
@@ -2,7 +2,7 @@ $:.push File.expand_path('lib', __dir__)
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = 'fluent-plugin-postgresql-csvlog'
|
5
|
-
s.version = '0.
|
5
|
+
s.version = '0.7.0'
|
6
6
|
s.authors = ['stanhu']
|
7
7
|
s.email = ['stanhu@gmail.com']
|
8
8
|
s.homepage = 'https://gitlab.com/gitlab-org/fluent-plugins/fluent-plugin-postgresql-csvlog'
|
@@ -19,6 +19,26 @@ module Fluent::Plugin
|
|
19
19
|
desc 'Name of field to store SQL query fingerprint'
|
20
20
|
config_param :fingerprint_key, :string, default: 'fingerprint'
|
21
21
|
|
22
|
+
POSTGRES_SERVER_VERSION_QUERY = "SELECT current_setting('server_version_num')"
|
23
|
+
|
24
|
+
PG12_STAT_STATEMENTS_QUERY = <<-SQL
|
25
|
+
SELECT queryid,
|
26
|
+
query,
|
27
|
+
calls,
|
28
|
+
rows,
|
29
|
+
total_time
|
30
|
+
FROM public.pg_stat_statements
|
31
|
+
SQL
|
32
|
+
|
33
|
+
PG13_STAT_STATEMENTS_QUERY = <<-SQL
|
34
|
+
SELECT queryid,
|
35
|
+
query,
|
36
|
+
calls,
|
37
|
+
rows,
|
38
|
+
(total_plan_time + total_exec_time) total_time
|
39
|
+
FROM public.pg_stat_statements
|
40
|
+
SQL
|
41
|
+
|
22
42
|
protected
|
23
43
|
|
24
44
|
def on_poll
|
@@ -29,13 +49,24 @@ module Fluent::Plugin
|
|
29
49
|
|
30
50
|
public
|
31
51
|
|
52
|
+
def initialize
|
53
|
+
super
|
54
|
+
@postgres_server_version_num = nil
|
55
|
+
end
|
56
|
+
|
32
57
|
# Returns a fluentd record for a query row
|
33
58
|
def record_for_row(row)
|
34
59
|
query = row['query']
|
35
60
|
|
36
61
|
# We record the query_length as it will help in understanding whether unparseable
|
37
62
|
# queries are truncated.
|
38
|
-
record = {
|
63
|
+
record = {
|
64
|
+
'queryid' => row['queryid'].to_s,
|
65
|
+
'query_length' => query&.length,
|
66
|
+
'calls' => row['calls']&.to_i,
|
67
|
+
'total_time_ms' => row['total_time']&.to_f,
|
68
|
+
'rows' => row['rows']&.to_i
|
69
|
+
}
|
39
70
|
|
40
71
|
return record unless query
|
41
72
|
|
@@ -56,12 +87,32 @@ module Fluent::Plugin
|
|
56
87
|
me = Fluent::MultiEventStream.new
|
57
88
|
|
58
89
|
now = Fluent::Engine.now
|
59
|
-
|
90
|
+
|
91
|
+
query = query_for_postgres_version(conn)
|
92
|
+
|
93
|
+
conn.exec(query).each do |row|
|
60
94
|
record = record_for_row(row)
|
61
95
|
me.add(now, record)
|
62
96
|
end
|
63
97
|
|
64
98
|
@router.emit_stream(@tag, me)
|
65
99
|
end
|
100
|
+
|
101
|
+
# Returns the PG_VERSION_NUM value from the database
|
102
|
+
# will memoize the result
|
103
|
+
def postgres_server_version_num(conn)
|
104
|
+
return @postgres_server_version_num if @postgres_server_version_num
|
105
|
+
|
106
|
+
@postgres_server_version_num = conn.exec(POSTGRES_SERVER_VERSION_QUERY).getvalue(0,0).to_i
|
107
|
+
end
|
108
|
+
|
109
|
+
# pg_stat_statements columns changed in pg13, so we use different queries depending on the version
|
110
|
+
# https://www.postgresql.org/docs/12/pgstatstatements.html
|
111
|
+
# https://www.postgresql.org/docs/13/pgstatstatements.html
|
112
|
+
def query_for_postgres_version(conn)
|
113
|
+
return PG13_STAT_STATEMENTS_QUERY if postgres_server_version_num(conn) >= 13_00_00
|
114
|
+
|
115
|
+
PG12_STAT_STATEMENTS_QUERY
|
116
|
+
end
|
66
117
|
end
|
67
118
|
end
|
@@ -42,13 +42,24 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
|
|
42
42
|
sub_test_case 'execution' do
|
43
43
|
test 'sql' do
|
44
44
|
d = create_driver
|
45
|
-
|
45
|
+
row = {
|
46
|
+
'queryid' => '1234',
|
47
|
+
'query' => 'SELECT * FROM users WHERE user_id = ?',
|
48
|
+
'calls' => 22,
|
49
|
+
'rows' => 333,
|
50
|
+
'total_time' => 44.44
|
51
|
+
}
|
52
|
+
|
53
|
+
record = d.instance.record_for_row(row)
|
46
54
|
|
47
55
|
expected = {
|
48
56
|
'fingerprint' => 'c071dee80d466e7d',
|
49
57
|
'query' => 'SELECT * FROM users WHERE user_id = ?',
|
50
58
|
'query_length' => 37,
|
51
|
-
'queryid' => '1234'
|
59
|
+
'queryid' => '1234',
|
60
|
+
'calls' => 22,
|
61
|
+
'rows' => 333,
|
62
|
+
'total_time_ms' => 44.44
|
52
63
|
}
|
53
64
|
|
54
65
|
assert_equal expected, record
|
@@ -56,9 +67,22 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
|
|
56
67
|
|
57
68
|
test 'nil query' do
|
58
69
|
d = create_driver
|
59
|
-
|
70
|
+
row = {
|
71
|
+
'queryid' => '1234',
|
72
|
+
'query' => nil,
|
73
|
+
'calls' => nil,
|
74
|
+
'rows' => nil,
|
75
|
+
'total_time' => nil
|
76
|
+
}
|
77
|
+
record = d.instance.record_for_row(row)
|
60
78
|
|
61
|
-
expected = {
|
79
|
+
expected = {
|
80
|
+
'query_length' => nil,
|
81
|
+
'queryid' => '1234',
|
82
|
+
'calls' => nil,
|
83
|
+
'rows' => nil,
|
84
|
+
'total_time_ms' => nil
|
85
|
+
}
|
62
86
|
assert_equal expected, record
|
63
87
|
end
|
64
88
|
|
@@ -75,26 +99,48 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
|
|
75
99
|
)
|
76
100
|
SQL
|
77
101
|
|
78
|
-
|
102
|
+
row = {
|
103
|
+
'queryid' => 1234,
|
104
|
+
'query' => ddl_sql,
|
105
|
+
'calls' => 22,
|
106
|
+
'rows' => 333,
|
107
|
+
'total_time' => 44.44
|
108
|
+
}
|
109
|
+
|
110
|
+
record = d.instance.record_for_row(row)
|
79
111
|
|
80
112
|
expected = {
|
81
113
|
'fingerprint' => 'fa9c9d26757c4f9b',
|
82
114
|
'query' => ddl_sql,
|
83
115
|
'query_length' => 287,
|
84
|
-
'queryid' => '1234'
|
116
|
+
'queryid' => '1234',
|
117
|
+
'calls' => 22,
|
118
|
+
'rows' => 333,
|
119
|
+
'total_time_ms' => 44.44
|
85
120
|
}
|
86
121
|
assert_equal expected, record
|
87
122
|
end
|
88
123
|
|
89
124
|
test 'set command' do
|
90
125
|
d = create_driver
|
91
|
-
|
126
|
+
row = {
|
127
|
+
'queryid' => 1234,
|
128
|
+
'query' => "SET TIME ZONE 'PST8PDT'",
|
129
|
+
'calls' => 22,
|
130
|
+
'rows' => 333,
|
131
|
+
'total_time' => 44.44
|
132
|
+
}
|
133
|
+
|
134
|
+
record = d.instance.record_for_row(row)
|
92
135
|
|
93
136
|
expected = {
|
94
137
|
'fingerprint' => '23f8d6eb1d3125c3',
|
95
138
|
'query' => 'SET TIME ZONE $1',
|
96
139
|
'query_length' => 23,
|
97
|
-
'queryid' => '1234'
|
140
|
+
'queryid' => '1234',
|
141
|
+
'calls' => 22,
|
142
|
+
'rows' => 333,
|
143
|
+
'total_time_ms' => 44.44
|
98
144
|
}
|
99
145
|
|
100
146
|
assert_equal expected, record
|
@@ -104,7 +150,15 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
|
|
104
150
|
d = create_driver
|
105
151
|
record = d.instance.record_for_row({ 'queryid' => 1234, 'query' => 'SELECT * FROM' })
|
106
152
|
|
107
|
-
expected = {
|
153
|
+
expected = {
|
154
|
+
'query_length' => 13,
|
155
|
+
'query_unparseable' => true,
|
156
|
+
'queryid' => '1234',
|
157
|
+
'calls' => nil,
|
158
|
+
'rows' => nil,
|
159
|
+
'total_time_ms' => nil
|
160
|
+
}
|
161
|
+
|
108
162
|
assert_equal expected, record
|
109
163
|
end
|
110
164
|
end
|
@@ -5,8 +5,8 @@
|
|
5
5
|
|
6
6
|
cleanup() {
|
7
7
|
echo "# removing all logs"
|
8
|
-
|
9
|
-
|
8
|
+
find /var/log/pg/ -name "pg_stat_statements.*.log" -delete
|
9
|
+
find /var/log/pg/ -name "pg_stat_activity.*.log" -delete
|
10
10
|
}
|
11
11
|
|
12
12
|
die() {
|
@@ -19,10 +19,14 @@ cleanup
|
|
19
19
|
echo "# sleeping 10, awaiting logs"
|
20
20
|
sleep 10;
|
21
21
|
|
22
|
-
|
23
|
-
cat /var/log/pg/pg_stat_statements.*.log | tail -1
|
22
|
+
echo "# looking for pg_stat_statements"
|
24
23
|
|
25
|
-
(find /var/log/pg/ -name "
|
26
|
-
cat /var/log/pg/
|
24
|
+
(find /var/log/pg/ -name "pg_stat_statements.*.log" | grep . >/dev/null) || die "No pg_stat_statements files created"
|
25
|
+
cat /var/log/pg/pg_stat_statements.*.log | tail -10
|
26
|
+
|
27
|
+
echo "# looking for pg_stat_activity"
|
28
|
+
|
29
|
+
(find /var/log/pg/ -name "pg_stat_activity.*.log" | grep . >/dev/null) || die "No pg_stat_activity files created"
|
30
|
+
cat /var/log/pg/pg_stat_activity.*.log | tail -10
|
27
31
|
|
28
32
|
cleanup
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-postgresql-csvlog
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- stanhu
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-12-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -125,7 +125,7 @@ files:
|
|
125
125
|
homepage: https://gitlab.com/gitlab-org/fluent-plugins/fluent-plugin-postgresql-csvlog
|
126
126
|
licenses: []
|
127
127
|
metadata: {}
|
128
|
-
post_install_message:
|
128
|
+
post_install_message:
|
129
129
|
rdoc_options: []
|
130
130
|
require_paths:
|
131
131
|
- lib
|
@@ -140,8 +140,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
140
140
|
- !ruby/object:Gem::Version
|
141
141
|
version: '0'
|
142
142
|
requirements: []
|
143
|
-
rubygems_version: 3.
|
144
|
-
signing_key:
|
143
|
+
rubygems_version: 3.2.28
|
144
|
+
signing_key:
|
145
145
|
specification_version: 4
|
146
146
|
summary: fluentd plugins to work with PostgreSQL CSV logs
|
147
147
|
test_files:
|