pg_metrics 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.markdown +14 -0
- data/README.markdown +21 -1
- data/lib/pg_metrics/metrics.rb +80 -3
- data/lib/pg_metrics/statsd.rb +33 -16
- data/lib/pg_metrics/version.rb +1 -1
- data/pg_metrics.gemspec +1 -1
- data/test/test_statsd.rb +26 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: baa62b576696fc88002fa07901d434d41ce5bfdf
|
4
|
+
data.tar.gz: e42fe1cfb12b58e4d20f63cf7408062c23bd589a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8d74d6b1239d90b66401e25862401d68ec3fe8ecdd7cc5de670eaea3485ccbae5fb10012b9f7d7cc6997df6ec3ff03f5d6adea439070f37e1cec30e8416b1e20
|
7
|
+
data.tar.gz: ca5d76a312f00266829cf8e6193093417769e17296690c5660b84f45a60ac8449700409ee456a8a020ca731f3433d8bba2132c902ed74581d1507f825ccd4d92
|
data/CHANGELOG.markdown
CHANGED
@@ -1,5 +1,19 @@
|
|
1
1
|
# pg_metrics Changelog
|
2
2
|
|
3
|
+
## Changes between 0.1.1 and 0.2.0
|
4
|
+
|
5
|
+
### Add --table-free-space and --index-ideal-size options
|
6
|
+
|
7
|
+
With `--table-free-space`, pg_metrics collects free space stats
|
8
|
+
(provided `pg_freespacemap` is installed). With `--index-ideal-sizes`,
|
9
|
+
`pg_metrics` estimates how large an index *should* be, assuming it has
|
10
|
+
no dead tuples. Both of these options are not included by default.
|
11
|
+
|
12
|
+
Motivated by the fact that `pg_freespace` in postgres versions 8.4
|
13
|
+
and up is quite slow and we don't want to run it as frequently as
|
14
|
+
other metrics, add an `--only` option that runs only the options
|
15
|
+
explicitly specified.
|
16
|
+
|
3
17
|
## Changes between 0.1.1 and 0.0.7
|
4
18
|
|
5
19
|
### Remove sensu
|
data/README.markdown
CHANGED
@@ -20,7 +20,7 @@ To collect PostgreSQL database metrics for the `prod` database, include the
|
|
20
20
|
|
21
21
|
pg_metrics_statsd --host localhost --port 8125 --connection "host=localhost port=5432" --dbname=prod
|
22
22
|
|
23
|
-
By default, pg_metrics_statsd collects stats from `pg_locks`,
|
23
|
+
By default, `pg_metrics_statsd` collects stats from `pg_locks`,
|
24
24
|
`pg_stat_user_functions` (where available), `pg_stat_user_tables`,
|
25
25
|
`pg_stat_user_tables`, `pg_stat_user_indexes`, `pg_statio_user_indexes`,
|
26
26
|
as well as per-table and per-index sizes. You can omit stats by supplying
|
@@ -35,6 +35,26 @@ command line flags:
|
|
35
35
|
- `--no-table-sizes`
|
36
36
|
- `--no-index-sizes`
|
37
37
|
|
38
|
+
You can also use the `--only` flag to collect *only* the explicitly specified
|
39
|
+
stats. For example, to collect only table and index size stats:
|
40
|
+
|
41
|
+
pg_metrics_statsd --only --table-sizes --index-sizes --dbname=prod
|
42
|
+
|
43
|
+
### Free space
|
44
|
+
`pg_metrics_statsd` can also collect table free space metrics provided the
|
45
|
+
`pg_freespacemap` contrib module is installed. To include free space metrics,
|
46
|
+
pass the `--table-free-space` flag along with the database. Collecting free
|
47
|
+
space metrics is not included by default.
|
48
|
+
|
49
|
+
For postgres versions 8.4 and higher, free space metrics take a while to generate.
|
50
|
+
|
51
|
+
### Index ideal sizes
|
52
|
+
`pg_metrics_statsd` includes an option to measure *ideal index size*, which is
|
53
|
+
an estimate of how much disk space an index *should* require if it contains no
|
54
|
+
dead tuples. This can be useful, along with index sizes, to estimate how bloated
|
55
|
+
indexes are. To collect ideal index size metrics, pass the `--index-ideal-sizes`
|
56
|
+
flag.
|
57
|
+
|
38
58
|
### pgbouncer metrics
|
39
59
|
|
40
60
|
`pg_metrics` can also collect `pgbouncer` metrics by passing the `--pgbouncer`
|
data/lib/pg_metrics/metrics.rb
CHANGED
@@ -12,6 +12,8 @@ module PgMetrics
|
|
12
12
|
TableStats = :table_stats
|
13
13
|
IndexStatio = :index_statio
|
14
14
|
IndexStats = :index_stats
|
15
|
+
TableFreeSpace = :table_free_space
|
16
|
+
IndexIdealSizes = :index_ideal_size
|
15
17
|
|
16
18
|
def self.fetch_instance_metrics(app_name, conn_info, regexp = nil)
|
17
19
|
metrics = []
|
@@ -231,7 +233,7 @@ array_to_string(ARRAY[funcname, '-', pronargs::TEXT,
|
|
231
233
|
calls, total_time, self_time
|
232
234
|
FROM pg_stat_user_functions
|
233
235
|
JOIN pg_proc ON pg_proc.oid = funcid
|
234
|
-
WHERE schemaname NOT IN ('information_schema', 'pg_catalog')) AS funcs}
|
236
|
+
WHERE schemaname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')) AS funcs}
|
235
237
|
: nil
|
236
238
|
},
|
237
239
|
|
@@ -262,7 +264,29 @@ array_to_string(ARRAY[funcname, '-', pronargs::TEXT,
|
|
262
264
|
FROM pg_class r
|
263
265
|
JOIN pg_namespace n ON r.relnamespace = n.oid
|
264
266
|
WHERE r.relkind = 'r'
|
265
|
-
AND n.nspname NOT IN ('pg_catalog', '
|
267
|
+
AND n.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')}
|
268
|
+
},
|
269
|
+
|
270
|
+
TableFreeSpace => {
|
271
|
+
prefix: %w(table),
|
272
|
+
query: Gem::Version.new(server_version) >= Gem::Version.new('8.4') \
|
273
|
+
? %{SELECT n.nspname AS key, t.relname AS key2,
|
274
|
+
COALESCE((SELECT sum(pg_freespace.avail) AS sum
|
275
|
+
FROM pg_freespace(t.oid::regclass) AS pg_freespace(blkno, avail)), 0::bigint) AS free_space
|
276
|
+
FROM pg_class t
|
277
|
+
JOIN pg_namespace n ON t.relnamespace = n.oid
|
278
|
+
WHERE t.relkind = 'r'::"char"
|
279
|
+
AND n.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')}
|
280
|
+
: %{SELECT n.nspname AS key, t.relname AS key2, fsm.bytes AS free_space
|
281
|
+
FROM pg_class t
|
282
|
+
JOIN pg_namespace n ON t.relnamespace = n.oid
|
283
|
+
LEFT JOIN (SELECT fsm.relfilenode, sum(fsm.bytes) AS bytes
|
284
|
+
FROM pg_freespacemap_pages fsm
|
285
|
+
JOIN pg_database db ON db.oid = fsm.reldatabase
|
286
|
+
AND db.datname = current_database()
|
287
|
+
GROUP BY fsm.relfilenode) fsm ON t.relfilenode = fsm.relfilenode
|
288
|
+
WHERE t.relkind = 'r'::"char"
|
289
|
+
AND n.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')}
|
266
290
|
},
|
267
291
|
|
268
292
|
IndexSizes => {
|
@@ -273,7 +297,60 @@ array_to_string(ARRAY[funcname, '-', pronargs::TEXT,
|
|
273
297
|
JOIN pg_class cr ON cr.oid = i.indrelid
|
274
298
|
JOIN pg_namespace n on ci.relnamespace = n.oid
|
275
299
|
WHERE ci.relkind = 'i' AND cr.relkind = 'r'
|
276
|
-
AND n.nspname NOT IN ('pg_catalog', '
|
300
|
+
AND n.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')}
|
301
|
+
},
|
302
|
+
|
303
|
+
IndexIdealSizes => {
|
304
|
+
prefix: %w(table),
|
305
|
+
query: %{SELECT pg_namespace.nspname AS key, rel.relname AS key2,
|
306
|
+
'index' AS key3, idx.relname AS key4,
|
307
|
+
((ceil(idx.reltuples * ((constants.index_tuple_header_size
|
308
|
+
+ constants.item_id_data_size
|
309
|
+
+ CASE WHEN (COALESCE(sum(CASE WHEN statts.staattnotnull THEN 0 ELSE 1 END), 0::bigint)
|
310
|
+
+ ((SELECT COALESCE(sum(CASE WHEN atts.attnotnull THEN 0 ELSE 1 END), 0::bigint) AS "coalesce"
|
311
|
+
FROM pg_attribute atts
|
312
|
+
JOIN (SELECT pg_index.indkey[the.i] AS attnum
|
313
|
+
FROM generate_series(0, pg_index.indnatts - 1) the(i)) cols ON atts.attnum = cols.attnum
|
314
|
+
WHERE atts.attrelid = pg_index.indrelid))) > 0
|
315
|
+
THEN (SELECT the.null_bitmap_size
|
316
|
+
+ constants.max_align
|
317
|
+
- CASE WHEN (the.null_bitmap_size % constants.max_align) = 0
|
318
|
+
THEN constants.max_align
|
319
|
+
ELSE the.null_bitmap_size % constants.max_align END
|
320
|
+
FROM (VALUES (ceil(pg_index.indnatts::real / 8)::int)) the (null_bitmap_size))
|
321
|
+
ELSE 0 END)::double precision
|
322
|
+
+ COALESCE(sum(statts.stawidth::double precision * (1::double precision - statts.stanullfrac)), 0::double precision)
|
323
|
+
+ COALESCE((SELECT sum(atts.stawidth::double precision * (1::double precision - atts.stanullfrac)) AS sum
|
324
|
+
FROM pg_statistic atts
|
325
|
+
JOIN (SELECT pg_index.indkey[the.i] AS attnum
|
326
|
+
FROM generate_series(0, pg_index.indnatts - 1) the(i)) cols ON atts.staattnum = cols.attnum
|
327
|
+
WHERE atts.starelid = pg_index.indrelid), 0::double precision))
|
328
|
+
/ (constants.block_size - constants.page_header_data_size::numeric - constants.special_space::numeric)::double precision)
|
329
|
+
+ constants.index_metadata_pages::double precision)
|
330
|
+
* constants.block_size::double precision)::bigint AS ideal_size
|
331
|
+
FROM pg_index
|
332
|
+
JOIN pg_class idx ON pg_index.indexrelid = idx.oid
|
333
|
+
JOIN pg_class rel ON pg_index.indrelid = rel.oid
|
334
|
+
JOIN pg_namespace ON idx.relnamespace = pg_namespace.oid
|
335
|
+
LEFT JOIN (SELECT pg_statistic.starelid, pg_statistic.staattnum, pg_statistic.stanullfrac, pg_statistic.stawidth, pg_attribute.attnotnull AS staattnotnull
|
336
|
+
FROM pg_statistic
|
337
|
+
JOIN pg_attribute ON (pg_statistic.starelid,pg_statistic.staattnum) = (pg_attribute.attrelid, pg_attribute.attnum)) statts
|
338
|
+
ON statts.starelid = idx.oid
|
339
|
+
CROSS JOIN (SELECT current_setting('block_size'::text)::numeric AS block_size,
|
340
|
+
CASE WHEN "substring"(version(), 12, 3) = ANY (ARRAY['8.0'::text, '8.1'::text, '8.2'::text]) THEN 27
|
341
|
+
ELSE 23 END AS tuple_header_size,
|
342
|
+
CASE WHEN version() ~ 'mingw32'::text THEN 8 ELSE 4 END AS max_align,
|
343
|
+
8 AS index_tuple_header_size,
|
344
|
+
4 AS item_id_data_size,
|
345
|
+
24 AS page_header_data_size,
|
346
|
+
0 AS special_space,
|
347
|
+
1 AS index_metadata_pages) AS constants
|
348
|
+
WHERE nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')
|
349
|
+
GROUP BY pg_namespace.nspname, rel.relname, rel.oid, idx.relname, idx.reltuples, idx.relpages, pg_index.indexrelid,
|
350
|
+
pg_index.indrelid, pg_index.indkey, pg_index.indnatts,
|
351
|
+
constants.block_size, constants.tuple_header_size, constants.max_align,
|
352
|
+
constants.index_tuple_header_size, constants.item_id_data_size, constants.page_header_data_size,
|
353
|
+
constants.index_metadata_pages, constants.special_space;}
|
277
354
|
},
|
278
355
|
|
279
356
|
TableStatio => {
|
data/lib/pg_metrics/statsd.rb
CHANGED
@@ -44,21 +44,24 @@ module PgMetrics
|
|
44
44
|
end
|
45
45
|
|
46
46
|
def self.parse(args)
|
47
|
+
default_stats = [PgMetrics::Metrics::Functions,
|
48
|
+
PgMetrics::Metrics::Locks,
|
49
|
+
PgMetrics::Metrics::TableSizes,
|
50
|
+
PgMetrics::Metrics::IndexSizes,
|
51
|
+
PgMetrics::Metrics::TableStatio,
|
52
|
+
PgMetrics::Metrics::TableStats,
|
53
|
+
PgMetrics::Metrics::IndexStatio,
|
54
|
+
PgMetrics::Metrics::IndexStats].to_set
|
47
55
|
options = {
|
48
56
|
host: "localhost",
|
49
57
|
port: 8125,
|
50
58
|
conn: "",
|
51
59
|
scheme: %(#{Socket.gethostname}.postgresql),
|
52
|
-
dbstats:
|
53
|
-
PgMetrics::Metrics::Locks,
|
54
|
-
PgMetrics::Metrics::TableSizes,
|
55
|
-
PgMetrics::Metrics::IndexSizes,
|
56
|
-
PgMetrics::Metrics::TableStatio,
|
57
|
-
PgMetrics::Metrics::TableStats,
|
58
|
-
PgMetrics::Metrics::IndexStatio,
|
59
|
-
PgMetrics::Metrics::IndexStats].to_set
|
60
|
+
dbstats: Set.new
|
60
61
|
}
|
61
62
|
|
63
|
+
stats = { added: Set.new, default: default_stats }
|
64
|
+
|
62
65
|
OptionParser.new do |opts|
|
63
66
|
opts.on("-h", "--host STATSD_HOST", "StatsD host") { |v| options[:host] = v }
|
64
67
|
opts.on("-p", "--port STATSD_PORT", "StatsD port") { |v| options[:port] = v.to_i }
|
@@ -66,20 +69,34 @@ module PgMetrics
|
|
66
69
|
opts.on("-d", "--dbname DBNAME", "PostgreSQL database name for database metrics") { |v| options[:dbname] = v }
|
67
70
|
opts.on("-e", "--exclude REGEXP", "Exclude objects matching given regexp") { |v| options[:exclude] = ::Regexp.new(v) }
|
68
71
|
opts.on("-s", "--scheme SCHEME", "Metric namespace") { |v| options[:scheme] = v }
|
69
|
-
opts.on("--
|
70
|
-
opts.on("--[no-]
|
71
|
-
opts.on("--[no-]
|
72
|
-
opts.on("--[no-]
|
73
|
-
opts.on("--[no-]
|
74
|
-
opts.on("--[no-]table-
|
75
|
-
opts.on("--[no-]
|
76
|
-
opts.on("--[no-]index-
|
72
|
+
opts.on("--only", "Collect only specified stats") { |v| stats[:default] = Set.new }
|
73
|
+
opts.on("--[no-]functions", "Collect database function stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::Functions, v) }
|
74
|
+
opts.on("--[no-]locks", "Collect database lock stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::Locks, v) }
|
75
|
+
opts.on("--[no-]table-sizes", "Collect database table size stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::TableSizes, v) }
|
76
|
+
opts.on("--[no-]index-sizes", "Collect database index size stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::IndexSizes, v) }
|
77
|
+
opts.on("--[no-]table-statio", "Collect database table statio stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::TableStatio, v) }
|
78
|
+
opts.on("--[no-]table-stats", "Collect database table stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::TableStats, v) }
|
79
|
+
opts.on("--[no-]index-statio", "Collect database index statio stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::IndexStatio, v) }
|
80
|
+
opts.on("--[no-]index-stats", "Collect database index stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::IndexStats, v) }
|
81
|
+
opts.on("--[no-]table-free-space", "Collect database table free space stats (requires pg_freespacemap)") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::TableFreeSpace, v) }
|
82
|
+
opts.on("--[no-]index-ideal-sizes", "Collect database index ideal size estimates") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::IndexIdealSizes, v) }
|
77
83
|
opts.on("--pgbouncer", "Collect pgbouncer stats") { |v| options[:pgbouncer] = true }
|
78
84
|
opts.on("--verbose") { |v| options[:verbose] = true }
|
79
85
|
opts.on("--version") { |v| options[:version] = v }
|
80
86
|
end.order!(args)
|
81
87
|
|
88
|
+
options[:dbstats] = stats[:added].merge(stats[:default])
|
82
89
|
options
|
83
90
|
end
|
91
|
+
|
92
|
+
def self.mutate_stats(stats, key, do_add)
|
93
|
+
if do_add
|
94
|
+
stats[:added].add(key)
|
95
|
+
else
|
96
|
+
stats[:default].delete(key)
|
97
|
+
end
|
98
|
+
stats
|
99
|
+
end
|
100
|
+
|
84
101
|
end
|
85
102
|
end
|
data/lib/pg_metrics/version.rb
CHANGED
data/pg_metrics.gemspec
CHANGED
@@ -7,7 +7,7 @@ Gem::Specification.new do |spec|
|
|
7
7
|
spec.name = "pg_metrics"
|
8
8
|
spec.version = PgMetrics::VERSION
|
9
9
|
spec.licenses = %w(MIT)
|
10
|
-
spec.date = "2015-03-
|
10
|
+
spec.date = "2015-03-19"
|
11
11
|
spec.summary = "pg_metrics"
|
12
12
|
spec.description = "PostgreSQL Metrics"
|
13
13
|
spec.authors = ["Michael Glaesemann"]
|
data/test/test_statsd.rb
CHANGED
@@ -53,7 +53,7 @@ module PgMetrics
|
|
53
53
|
PgMetrics::Metrics::TableStats,
|
54
54
|
PgMetrics::Metrics::IndexStatio,
|
55
55
|
PgMetrics::Metrics::IndexStats].to_set
|
56
|
-
assert_equal(config[:dbstats]
|
56
|
+
assert_equal(expected, config[:dbstats])
|
57
57
|
end
|
58
58
|
|
59
59
|
def test_should_set_all_metrics_with_positive_locks
|
@@ -67,7 +67,7 @@ module PgMetrics
|
|
67
67
|
PgMetrics::Metrics::TableStats,
|
68
68
|
PgMetrics::Metrics::IndexStatio,
|
69
69
|
PgMetrics::Metrics::IndexStats].to_set
|
70
|
-
assert_equal(config[:dbstats]
|
70
|
+
assert_equal(expected, config[:dbstats])
|
71
71
|
end
|
72
72
|
|
73
73
|
def test_should_not_collect_locks
|
@@ -80,14 +80,36 @@ module PgMetrics
|
|
80
80
|
PgMetrics::Metrics::TableStats,
|
81
81
|
PgMetrics::Metrics::IndexStatio,
|
82
82
|
PgMetrics::Metrics::IndexStats].to_set
|
83
|
-
assert_equal(config[:dbstats]
|
83
|
+
assert_equal(expected, config[:dbstats])
|
84
84
|
end
|
85
85
|
|
86
86
|
def test_should_remove_all_but_locks
|
87
87
|
args = %w(--no-functions --no-table-sizes --no-index-sizes --no-table-statio --no-table-stats --no-index-stats --no-index-statio)
|
88
88
|
config = PgMetrics::Statsd::parse(args)
|
89
89
|
expected = [PgMetrics::Metrics::Locks].to_set
|
90
|
-
assert_equal(config[:dbstats]
|
90
|
+
assert_equal(expected, config[:dbstats])
|
91
|
+
end
|
92
|
+
|
93
|
+
def test_should_removal_all_but_locks_using_only
|
94
|
+
args = %w(--only --locks)
|
95
|
+
config = PgMetrics::Statsd::parse(args)
|
96
|
+
expected = [PgMetrics::Metrics::Locks].to_set
|
97
|
+
assert_equal(expected, config[:dbstats])
|
98
|
+
end
|
99
|
+
|
100
|
+
def test_should_only_include_table_free_space
|
101
|
+
args = %w(--only --table-free-space)
|
102
|
+
config = PgMetrics::Statsd::parse(args)
|
103
|
+
expected = [PgMetrics::Metrics::TableFreeSpace].to_set
|
104
|
+
assert_equal(expected, config[:dbstats])
|
105
|
+
end
|
106
|
+
|
107
|
+
def test_should_only_include_table_free_space
|
108
|
+
args = %w(--only --index-ideal-sizes --table-free-space)
|
109
|
+
config = PgMetrics::Statsd::parse(args)
|
110
|
+
expected = [PgMetrics::Metrics::TableFreeSpace,
|
111
|
+
PgMetrics::Metrics::IndexIdealSizes].to_set
|
112
|
+
assert_equal(expected, config[:dbstats])
|
91
113
|
end
|
92
114
|
|
93
115
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_metrics
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Glaesemann
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-03-
|
11
|
+
date: 2015-03-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pg
|