pg_metrics 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 1f02f742e23e3d02686a15c2cf6a9f8b48b527e3
4
- data.tar.gz: cb0e25e46b852b7cc3f3551b3c87d63bbf187acb
3
+ metadata.gz: baa62b576696fc88002fa07901d434d41ce5bfdf
4
+ data.tar.gz: e42fe1cfb12b58e4d20f63cf7408062c23bd589a
5
5
  SHA512:
6
- metadata.gz: 615b4a325678b94b2f3596f924c8a5ac978314d2a4b0a18d09eb6785d45882545291948298b2ba849f20efd4b4e075cda241d7b091584a5ac1f60c656296f7df
7
- data.tar.gz: 455211aeb1ef3fa4883edbee01bc70d9111433dfe9e04950134765097a8f7664d131b31e1f6b26e3e24a46e3422e6146a85ffc482bdf01c2481de26a8b9dc236
6
+ metadata.gz: 8d74d6b1239d90b66401e25862401d68ec3fe8ecdd7cc5de670eaea3485ccbae5fb10012b9f7d7cc6997df6ec3ff03f5d6adea439070f37e1cec30e8416b1e20
7
+ data.tar.gz: ca5d76a312f00266829cf8e6193093417769e17296690c5660b84f45a60ac8449700409ee456a8a020ca731f3433d8bba2132c902ed74581d1507f825ccd4d92
data/CHANGELOG.markdown CHANGED
@@ -1,5 +1,19 @@
1
1
  # pg_metrics Changelog
2
2
 
3
+ ## Changes between 0.1.1 and 0.2.0
4
+
5
+ ### Add --table-free-space and --index-ideal-size options
6
+
7
+ With `--table-free-space`, pg_metrics collects free space stats
8
+ (provided `pg_freespacemap` is installed). With `--index-ideal-sizes`,
9
+ `pg_metrics` estimates how large an index *should* be, assuming it has
10
+ no dead tuples. Both of these options are not included by default.
11
+
12
+ Motivated by the fact that `pg_freespace` in postgres versions 8.4
13
+ and up is quite slow and we don't want to run it as frequently as
14
+ other metrics, add an `--only` option that runs only the options
15
+ explicitly specified.
16
+
3
17
  ## Changes between 0.1.1 and 0.0.7
4
18
 
5
19
  ### Remove sensu
data/README.markdown CHANGED
@@ -20,7 +20,7 @@ To collect PostgreSQL database metrics for the `prod` database, include the
20
20
 
21
21
  pg_metrics_statsd --host localhost --port 8125 --connection "host=localhost port=5432" --dbname=prod
22
22
 
23
- By default, pg_metrics_statsd collects stats from `pg_locks`,
23
+ By default, `pg_metrics_statsd` collects stats from `pg_locks`,
24
24
  `pg_stat_user_functions` (where available), `pg_stat_user_tables`,
25
25
  `pg_stat_user_tables`, `pg_stat_user_indexes`, `pg_statio_user_indexes`,
26
26
  as well as per-table and per-index sizes. You can omit stats by supplying
@@ -35,6 +35,26 @@ command line flags:
35
35
  - `--no-table-sizes`
36
36
  - `--no-index-sizes`
37
37
 
38
+ You can also use the `--only` flag to collect *only* the explicitly specified
39
+ stats. For example, to collect only table and index size stats:
40
+
41
+ pg_metrics_statsd --only --table-sizes --index-sizes --dbname=prod
42
+
43
+ ### Free space
44
+ `pg_metrics_statsd` can also collect table free space metrics provided the
45
+ `pg_freespacemap` contrib module is installed. To include free space metrics,
46
+ pass the `--table-free-space` flag along with the database. Collecting free
47
+ space metrics is not included by default.
48
+
49
+ For postgres versions 8.4 and higher, free space metrics take a while to generate.
50
+
51
+ ### Index ideal sizes
52
+ `pg_metrics_statsd` includes an option to measure *ideal index size*, which is
53
+ an estimate of how much disk space an index *should* require if it contains no
54
+ dead tuples. This can be useful, along with index sizes, to estimate how bloated
55
+ indexes are. To collect ideal index size metrics, pass the `--index-ideal-sizes`
56
+ flag.
57
+
38
58
  ### pgbouncer metrics
39
59
 
40
60
  `pg_metrics` can also collect `pgbouncer` metrics by passing the `--pgbouncer`
@@ -12,6 +12,8 @@ module PgMetrics
12
12
  TableStats = :table_stats
13
13
  IndexStatio = :index_statio
14
14
  IndexStats = :index_stats
15
+ TableFreeSpace = :table_free_space
16
+ IndexIdealSizes = :index_ideal_size
15
17
 
16
18
  def self.fetch_instance_metrics(app_name, conn_info, regexp = nil)
17
19
  metrics = []
@@ -231,7 +233,7 @@ array_to_string(ARRAY[funcname, '-', pronargs::TEXT,
231
233
  calls, total_time, self_time
232
234
  FROM pg_stat_user_functions
233
235
  JOIN pg_proc ON pg_proc.oid = funcid
234
- WHERE schemaname NOT IN ('information_schema', 'pg_catalog')) AS funcs}
236
+ WHERE schemaname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')) AS funcs}
235
237
  : nil
236
238
  },
237
239
 
@@ -262,7 +264,29 @@ array_to_string(ARRAY[funcname, '-', pronargs::TEXT,
262
264
  FROM pg_class r
263
265
  JOIN pg_namespace n ON r.relnamespace = n.oid
264
266
  WHERE r.relkind = 'r'
265
- AND n.nspname NOT IN ('pg_catalog', 'information_schema')}
267
+ AND n.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')}
268
+ },
269
+
270
+ TableFreeSpace => {
271
+ prefix: %w(table),
272
+ query: Gem::Version.new(server_version) >= Gem::Version.new('8.4') \
273
+ ? %{SELECT n.nspname AS key, t.relname AS key2,
274
+ COALESCE((SELECT sum(pg_freespace.avail) AS sum
275
+ FROM pg_freespace(t.oid::regclass) AS pg_freespace(blkno, avail)), 0::bigint) AS free_space
276
+ FROM pg_class t
277
+ JOIN pg_namespace n ON t.relnamespace = n.oid
278
+ WHERE t.relkind = 'r'::"char"
279
+ AND n.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')}
280
+ : %{SELECT n.nspname AS key, t.relname AS key2, fsm.bytes AS free_space
281
+ FROM pg_class t
282
+ JOIN pg_namespace n ON t.relnamespace = n.oid
283
+ LEFT JOIN (SELECT fsm.relfilenode, sum(fsm.bytes) AS bytes
284
+ FROM pg_freespacemap_pages fsm
285
+ JOIN pg_database db ON db.oid = fsm.reldatabase
286
+ AND db.datname = current_database()
287
+ GROUP BY fsm.relfilenode) fsm ON t.relfilenode = fsm.relfilenode
288
+ WHERE t.relkind = 'r'::"char"
289
+ AND n.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')}
266
290
  },
267
291
 
268
292
  IndexSizes => {
@@ -273,7 +297,60 @@ array_to_string(ARRAY[funcname, '-', pronargs::TEXT,
273
297
  JOIN pg_class cr ON cr.oid = i.indrelid
274
298
  JOIN pg_namespace n on ci.relnamespace = n.oid
275
299
  WHERE ci.relkind = 'i' AND cr.relkind = 'r'
276
- AND n.nspname NOT IN ('pg_catalog', 'information_schema')}
300
+ AND n.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')}
301
+ },
302
+
303
+ IndexIdealSizes => {
304
+ prefix: %w(table),
305
+ query: %{SELECT pg_namespace.nspname AS key, rel.relname AS key2,
306
+ 'index' AS key3, idx.relname AS key4,
307
+ ((ceil(idx.reltuples * ((constants.index_tuple_header_size
308
+ + constants.item_id_data_size
309
+ + CASE WHEN (COALESCE(sum(CASE WHEN statts.staattnotnull THEN 0 ELSE 1 END), 0::bigint)
310
+ + ((SELECT COALESCE(sum(CASE WHEN atts.attnotnull THEN 0 ELSE 1 END), 0::bigint) AS "coalesce"
311
+ FROM pg_attribute atts
312
+ JOIN (SELECT pg_index.indkey[the.i] AS attnum
313
+ FROM generate_series(0, pg_index.indnatts - 1) the(i)) cols ON atts.attnum = cols.attnum
314
+ WHERE atts.attrelid = pg_index.indrelid))) > 0
315
+ THEN (SELECT the.null_bitmap_size
316
+ + constants.max_align
317
+ - CASE WHEN (the.null_bitmap_size % constants.max_align) = 0
318
+ THEN constants.max_align
319
+ ELSE the.null_bitmap_size % constants.max_align END
320
+ FROM (VALUES (ceil(pg_index.indnatts::real / 8)::int)) the (null_bitmap_size))
321
+ ELSE 0 END)::double precision
322
+ + COALESCE(sum(statts.stawidth::double precision * (1::double precision - statts.stanullfrac)), 0::double precision)
323
+ + COALESCE((SELECT sum(atts.stawidth::double precision * (1::double precision - atts.stanullfrac)) AS sum
324
+ FROM pg_statistic atts
325
+ JOIN (SELECT pg_index.indkey[the.i] AS attnum
326
+ FROM generate_series(0, pg_index.indnatts - 1) the(i)) cols ON atts.staattnum = cols.attnum
327
+ WHERE atts.starelid = pg_index.indrelid), 0::double precision))
328
+ / (constants.block_size - constants.page_header_data_size::numeric - constants.special_space::numeric)::double precision)
329
+ + constants.index_metadata_pages::double precision)
330
+ * constants.block_size::double precision)::bigint AS ideal_size
331
+ FROM pg_index
332
+ JOIN pg_class idx ON pg_index.indexrelid = idx.oid
333
+ JOIN pg_class rel ON pg_index.indrelid = rel.oid
334
+ JOIN pg_namespace ON idx.relnamespace = pg_namespace.oid
335
+ LEFT JOIN (SELECT pg_statistic.starelid, pg_statistic.staattnum, pg_statistic.stanullfrac, pg_statistic.stawidth, pg_attribute.attnotnull AS staattnotnull
336
+ FROM pg_statistic
337
+ JOIN pg_attribute ON (pg_statistic.starelid,pg_statistic.staattnum) = (pg_attribute.attrelid, pg_attribute.attnum)) statts
338
+ ON statts.starelid = idx.oid
339
+ CROSS JOIN (SELECT current_setting('block_size'::text)::numeric AS block_size,
340
+ CASE WHEN "substring"(version(), 12, 3) = ANY (ARRAY['8.0'::text, '8.1'::text, '8.2'::text]) THEN 27
341
+ ELSE 23 END AS tuple_header_size,
342
+ CASE WHEN version() ~ 'mingw32'::text THEN 8 ELSE 4 END AS max_align,
343
+ 8 AS index_tuple_header_size,
344
+ 4 AS item_id_data_size,
345
+ 24 AS page_header_data_size,
346
+ 0 AS special_space,
347
+ 1 AS index_metadata_pages) AS constants
348
+ WHERE nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')
349
+ GROUP BY pg_namespace.nspname, rel.relname, rel.oid, idx.relname, idx.reltuples, idx.relpages, pg_index.indexrelid,
350
+ pg_index.indrelid, pg_index.indkey, pg_index.indnatts,
351
+ constants.block_size, constants.tuple_header_size, constants.max_align,
352
+ constants.index_tuple_header_size, constants.item_id_data_size, constants.page_header_data_size,
353
+ constants.index_metadata_pages, constants.special_space;}
277
354
  },
278
355
 
279
356
  TableStatio => {
@@ -44,21 +44,24 @@ module PgMetrics
44
44
  end
45
45
 
46
46
  def self.parse(args)
47
+ default_stats = [PgMetrics::Metrics::Functions,
48
+ PgMetrics::Metrics::Locks,
49
+ PgMetrics::Metrics::TableSizes,
50
+ PgMetrics::Metrics::IndexSizes,
51
+ PgMetrics::Metrics::TableStatio,
52
+ PgMetrics::Metrics::TableStats,
53
+ PgMetrics::Metrics::IndexStatio,
54
+ PgMetrics::Metrics::IndexStats].to_set
47
55
  options = {
48
56
  host: "localhost",
49
57
  port: 8125,
50
58
  conn: "",
51
59
  scheme: %(#{Socket.gethostname}.postgresql),
52
- dbstats: [PgMetrics::Metrics::Functions,
53
- PgMetrics::Metrics::Locks,
54
- PgMetrics::Metrics::TableSizes,
55
- PgMetrics::Metrics::IndexSizes,
56
- PgMetrics::Metrics::TableStatio,
57
- PgMetrics::Metrics::TableStats,
58
- PgMetrics::Metrics::IndexStatio,
59
- PgMetrics::Metrics::IndexStats].to_set
60
+ dbstats: Set.new
60
61
  }
61
62
 
63
+ stats = { added: Set.new, default: default_stats }
64
+
62
65
  OptionParser.new do |opts|
63
66
  opts.on("-h", "--host STATSD_HOST", "StatsD host") { |v| options[:host] = v }
64
67
  opts.on("-p", "--port STATSD_PORT", "StatsD port") { |v| options[:port] = v.to_i }
@@ -66,20 +69,34 @@ module PgMetrics
66
69
  opts.on("-d", "--dbname DBNAME", "PostgreSQL database name for database metrics") { |v| options[:dbname] = v }
67
70
  opts.on("-e", "--exclude REGEXP", "Exclude objects matching given regexp") { |v| options[:exclude] = ::Regexp.new(v) }
68
71
  opts.on("-s", "--scheme SCHEME", "Metric namespace") { |v| options[:scheme] = v }
69
- opts.on("--[no-]functions", "Collect database function stats") { |v| options[:dbstats].delete(PgMetrics::Metrics::Functions) unless v }
70
- opts.on("--[no-]locks", "Collect database lock stats") { |v| options[:dbstats].delete(PgMetrics::Metrics::Locks) unless v }
71
- opts.on("--[no-]table-sizes", "Collect database table size stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::TableSizes) unless v }
72
- opts.on("--[no-]index-sizes", "Collect database index size stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::IndexSizes) unless v }
73
- opts.on("--[no-]table-statio", "Collect database table statio stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::TableStatio) unless v }
74
- opts.on("--[no-]table-stats", "Collect database table stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::TableStats) unless v }
75
- opts.on("--[no-]index-statio", "Collect database index statio stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::IndexStatio) unless v }
76
- opts.on("--[no-]index-stats", "Collect database index stats ") { |v| options[:dbstats].delete(PgMetrics::Metrics::IndexStats) unless v }
72
+ opts.on("--only", "Collect only specified stats") { |v| stats[:default] = Set.new }
73
+ opts.on("--[no-]functions", "Collect database function stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::Functions, v) }
74
+ opts.on("--[no-]locks", "Collect database lock stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::Locks, v) }
75
+ opts.on("--[no-]table-sizes", "Collect database table size stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::TableSizes, v) }
76
+ opts.on("--[no-]index-sizes", "Collect database index size stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::IndexSizes, v) }
77
+ opts.on("--[no-]table-statio", "Collect database table statio stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::TableStatio, v) }
78
+ opts.on("--[no-]table-stats", "Collect database table stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::TableStats, v) }
79
+ opts.on("--[no-]index-statio", "Collect database index statio stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::IndexStatio, v) }
80
+ opts.on("--[no-]index-stats", "Collect database index stats") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::IndexStats, v) }
81
+ opts.on("--[no-]table-free-space", "Collect database table free space stats (requires pg_freespacemap)") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::TableFreeSpace, v) }
82
+ opts.on("--[no-]index-ideal-sizes", "Collect database index ideal size estimates") { |v| stats = mutate_stats(stats, PgMetrics::Metrics::IndexIdealSizes, v) }
77
83
  opts.on("--pgbouncer", "Collect pgbouncer stats") { |v| options[:pgbouncer] = true }
78
84
  opts.on("--verbose") { |v| options[:verbose] = true }
79
85
  opts.on("--version") { |v| options[:version] = v }
80
86
  end.order!(args)
81
87
 
88
+ options[:dbstats] = stats[:added].merge(stats[:default])
82
89
  options
83
90
  end
91
+
92
+ def self.mutate_stats(stats, key, do_add)
93
+ if do_add
94
+ stats[:added].add(key)
95
+ else
96
+ stats[:default].delete(key)
97
+ end
98
+ stats
99
+ end
100
+
84
101
  end
85
102
  end
@@ -1,3 +1,3 @@
1
1
  module PgMetrics
2
- VERSION = "0.1.1"
2
+ VERSION = "0.2.0"
3
3
  end
data/pg_metrics.gemspec CHANGED
@@ -7,7 +7,7 @@ Gem::Specification.new do |spec|
7
7
  spec.name = "pg_metrics"
8
8
  spec.version = PgMetrics::VERSION
9
9
  spec.licenses = %w(MIT)
10
- spec.date = "2015-03-17"
10
+ spec.date = "2015-03-19"
11
11
  spec.summary = "pg_metrics"
12
12
  spec.description = "PostgreSQL Metrics"
13
13
  spec.authors = ["Michael Glaesemann"]
data/test/test_statsd.rb CHANGED
@@ -53,7 +53,7 @@ module PgMetrics
53
53
  PgMetrics::Metrics::TableStats,
54
54
  PgMetrics::Metrics::IndexStatio,
55
55
  PgMetrics::Metrics::IndexStats].to_set
56
- assert_equal(config[:dbstats], expected)
56
+ assert_equal(expected, config[:dbstats])
57
57
  end
58
58
 
59
59
  def test_should_set_all_metrics_with_positive_locks
@@ -67,7 +67,7 @@ module PgMetrics
67
67
  PgMetrics::Metrics::TableStats,
68
68
  PgMetrics::Metrics::IndexStatio,
69
69
  PgMetrics::Metrics::IndexStats].to_set
70
- assert_equal(config[:dbstats], expected)
70
+ assert_equal(expected, config[:dbstats])
71
71
  end
72
72
 
73
73
  def test_should_not_collect_locks
@@ -80,14 +80,36 @@ module PgMetrics
80
80
  PgMetrics::Metrics::TableStats,
81
81
  PgMetrics::Metrics::IndexStatio,
82
82
  PgMetrics::Metrics::IndexStats].to_set
83
- assert_equal(config[:dbstats], expected)
83
+ assert_equal(expected, config[:dbstats])
84
84
  end
85
85
 
86
86
  def test_should_remove_all_but_locks
87
87
  args = %w(--no-functions --no-table-sizes --no-index-sizes --no-table-statio --no-table-stats --no-index-stats --no-index-statio)
88
88
  config = PgMetrics::Statsd::parse(args)
89
89
  expected = [PgMetrics::Metrics::Locks].to_set
90
- assert_equal(config[:dbstats], expected)
90
+ assert_equal(expected, config[:dbstats])
91
+ end
92
+
93
+ def test_should_removal_all_but_locks_using_only
94
+ args = %w(--only --locks)
95
+ config = PgMetrics::Statsd::parse(args)
96
+ expected = [PgMetrics::Metrics::Locks].to_set
97
+ assert_equal(expected, config[:dbstats])
98
+ end
99
+
100
+ def test_should_only_include_table_free_space
101
+ args = %w(--only --table-free-space)
102
+ config = PgMetrics::Statsd::parse(args)
103
+ expected = [PgMetrics::Metrics::TableFreeSpace].to_set
104
+ assert_equal(expected, config[:dbstats])
105
+ end
106
+
107
+ def test_should_only_include_table_free_space
108
+ args = %w(--only --index-ideal-sizes --table-free-space)
109
+ config = PgMetrics::Statsd::parse(args)
110
+ expected = [PgMetrics::Metrics::TableFreeSpace,
111
+ PgMetrics::Metrics::IndexIdealSizes].to_set
112
+ assert_equal(expected, config[:dbstats])
91
113
  end
92
114
 
93
115
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pg_metrics
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Michael Glaesemann
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-03-17 00:00:00.000000000 Z
11
+ date: 2015-03-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: pg