pg_reports 0.9.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +82 -0
- data/README.md +12 -8
- data/app/controllers/pg_reports/dashboard_controller.rb +277 -221
- data/app/controllers/pg_reports/metrics_controller.rb +13 -4
- data/app/views/layouts/pg_reports/application.html.erb +222 -93
- data/app/views/pg_reports/dashboard/_settings_modal.html.erb +55 -0
- data/app/views/pg_reports/dashboard/_show_modals.html.erb +1 -42
- data/app/views/pg_reports/dashboard/_show_scripts.html.erb +75 -105
- data/app/views/pg_reports/dashboard/index.html.erb +346 -238
- data/app/views/pg_reports/dashboard/show.html.erb +1 -1
- data/bin/pg_reports +2 -2
- data/config/brakeman.ignore +10 -56
- data/config/locales/en.yml +45 -11
- data/config/locales/ru.yml +44 -11
- data/config/locales/uk.yml +44 -11
- data/config/routes.rb +2 -0
- data/lib/pg_reports/access_policy.rb +39 -0
- data/lib/pg_reports/compatibility.rb +10 -2
- data/lib/pg_reports/configuration.rb +33 -2
- data/lib/pg_reports/connection/error_translator.rb +21 -0
- data/lib/pg_reports/connection/target.rb +43 -10
- data/lib/pg_reports/csv_cell.rb +20 -0
- data/lib/pg_reports/dashboard/reports_registry.rb +37 -30
- data/lib/pg_reports/definitions/connections/connection_stats.yml +4 -0
- data/lib/pg_reports/definitions/connections/idle_connections.yml +4 -0
- data/lib/pg_reports/definitions/connections/locks.yml +1 -0
- data/lib/pg_reports/definitions/connections/long_running_queries.yml +0 -1
- data/lib/pg_reports/definitions/indexes/bloated_indexes.yml +1 -2
- data/lib/pg_reports/definitions/indexes/index_correlation.yml +2 -1
- data/lib/pg_reports/definitions/indexes/missing_indexes.yml +1 -0
- data/lib/pg_reports/definitions/queries/frequent_queries.yml +50 -0
- data/lib/pg_reports/definitions/system/cache_stats.yml +2 -1
- data/lib/pg_reports/definitions/tables/bloated_tables.yml +1 -2
- data/lib/pg_reports/definitions/tables/recently_modified.yml +3 -3
- data/lib/pg_reports/definitions/tables/unused_tables.yml +1 -1
- data/lib/pg_reports/drop_index_migration.rb +56 -0
- data/lib/pg_reports/engine.rb +4 -3
- data/lib/pg_reports/executor.rb +46 -9
- data/lib/pg_reports/modules/queries.rb +1 -0
- data/lib/pg_reports/query_monitor.rb +478 -192
- data/lib/pg_reports/rate_limiter.rb +63 -0
- data/lib/pg_reports/report.rb +1 -1
- data/lib/pg_reports/report_definition.rb +11 -1
- data/lib/pg_reports/sql/connections/connection_stats.sql +2 -1
- data/lib/pg_reports/sql/connections/idle_connections.sql +1 -0
- data/lib/pg_reports/sql/connections/locks.sql +1 -0
- data/lib/pg_reports/sql/indexes/bloated_indexes.sql +16 -8
- data/lib/pg_reports/sql/queries/activity_sample.sql +22 -0
- data/lib/pg_reports/sql/queries/frequent_queries.sql +42 -0
- data/lib/pg_reports/sql_redactor.rb +32 -0
- data/lib/pg_reports/standalone.rb +83 -20
- data/lib/pg_reports/version.rb +1 -1
- data/lib/pg_reports.rb +7 -2
- metadata +18 -9
|
@@ -9,6 +9,7 @@ module PgReports
|
|
|
9
9
|
# Query analysis thresholds
|
|
10
10
|
attr_accessor :slow_query_threshold_ms # Queries slower than this are considered slow
|
|
11
11
|
attr_accessor :heavy_query_threshold_calls # Queries with more calls than this are heavy
|
|
12
|
+
attr_accessor :frequent_query_threshold_per_second # Queries at or above this rate are frequent
|
|
12
13
|
attr_accessor :expensive_query_threshold_ms # Total time threshold for expensive queries
|
|
13
14
|
|
|
14
15
|
# Index analysis thresholds
|
|
@@ -26,7 +27,7 @@ module PgReports
|
|
|
26
27
|
attr_accessor :max_query_length # Truncate query text to this length
|
|
27
28
|
|
|
28
29
|
# Dashboard settings
|
|
29
|
-
attr_accessor :dashboard_auth # Proc
|
|
30
|
+
attr_accessor :dashboard_auth # Proc run as a before_action, or :host_managed. Unset: loopback only, never in production
|
|
30
31
|
attr_accessor :standalone # True when running via PgReports::Standalone (no host app)
|
|
31
32
|
|
|
32
33
|
# Assets / privacy settings
|
|
@@ -36,7 +37,13 @@ module PgReports
|
|
|
36
37
|
attr_accessor :fake_source_data # Inject fake source data for IDE link testing
|
|
37
38
|
|
|
38
39
|
# Query monitoring settings
|
|
40
|
+
attr_accessor :query_monitor_enabled # Off in production by default; false removes the monitor entirely
|
|
41
|
+
attr_accessor :query_monitor_sources # Capture methods on offer: :notifications, :activity
|
|
42
|
+
attr_accessor :query_monitor_cache_store # Cache shared by all processes (default: Rails.cache)
|
|
43
|
+
attr_accessor :query_monitor_max_duration # Seconds after which a session stops by itself
|
|
39
44
|
attr_accessor :query_monitor_log_file # Path to log file for query monitoring
|
|
45
|
+
attr_accessor :query_monitor_log_max_bytes # Size at which the log is rotated to <file>.1. nil or 0 disables rotation.
|
|
46
|
+
attr_accessor :query_monitor_redact_literals # Replace string literals in captured SQL with '?'. On outside development.
|
|
40
47
|
attr_accessor :query_monitor_max_queries # Maximum number of queries to keep in buffer
|
|
41
48
|
attr_accessor :query_monitor_backtrace_filter # Proc to filter backtrace lines
|
|
42
49
|
|
|
@@ -44,12 +51,13 @@ module PgReports
|
|
|
44
51
|
attr_accessor :allow_raw_query_execution # Allow execute_query and explain_analyze from dashboard
|
|
45
52
|
attr_accessor :allow_migration_creation # Allow dashboard's "Generate Migration" button to write files into db/migrate/
|
|
46
53
|
attr_accessor :raw_query_statement_timeout_ms # Postgres statement_timeout (ms) applied to Execute Query / EXPLAIN ANALYZE / SQL Console. 0 disables it.
|
|
54
|
+
attr_accessor :report_statement_timeout_ms # Postgres statement_timeout (ms) for the SQL behind reports and live metrics. 0 disables it.
|
|
47
55
|
attr_accessor :raw_query_rate_limit # Max privileged requests (raw query execution / Generate Migration) per client IP per window. nil disables rate limiting.
|
|
48
56
|
attr_accessor :raw_query_rate_limit_window_seconds # Window size (seconds) for raw_query_rate_limit
|
|
49
57
|
|
|
50
58
|
# Grafana / Prometheus exporter settings
|
|
51
59
|
attr_accessor :grafana_favorites # Reports exposed at /metrics (Array of keys or Hash with per-report opts)
|
|
52
|
-
attr_accessor :grafana_metrics_token # Bearer token required to access /metrics
|
|
60
|
+
attr_accessor :grafana_metrics_token # Bearer token required to access /metrics, or :host_managed. Unset: as dashboard_auth
|
|
53
61
|
attr_accessor :grafana_cache_ttl # Default cache TTL for collected reports, seconds
|
|
54
62
|
|
|
55
63
|
def initialize
|
|
@@ -60,6 +68,7 @@ module PgReports
|
|
|
60
68
|
# Query thresholds
|
|
61
69
|
@slow_query_threshold_ms = 100
|
|
62
70
|
@heavy_query_threshold_calls = 1000
|
|
71
|
+
@frequent_query_threshold_per_second = 1
|
|
63
72
|
@expensive_query_threshold_ms = 10_000
|
|
64
73
|
|
|
65
74
|
# Index thresholds
|
|
@@ -87,7 +96,19 @@ module PgReports
|
|
|
87
96
|
@fake_source_data = ENV.fetch("PG_REPORTS_FAKE_SOURCE_DATA", "false") == "true"
|
|
88
97
|
|
|
89
98
|
# Query monitoring
|
|
99
|
+
rails_env = Rails.env if defined?(Rails) && Rails.respond_to?(:env)
|
|
100
|
+
# Off in production, so that a deploy adds nothing to the application's
|
|
101
|
+
# queries until someone decides they want the monitor there
|
|
102
|
+
@query_monitor_enabled = !rails_env&.production?
|
|
103
|
+
# Two sources are a choice for a developer; production keeps to one
|
|
104
|
+
@query_monitor_sources = rails_env&.development? ? %i[notifications activity] : %i[notifications]
|
|
105
|
+
@query_monitor_cache_store = nil
|
|
106
|
+
@query_monitor_max_duration = 900
|
|
90
107
|
@query_monitor_log_file = defined?(Rails) ? Rails.root.join("log", "pg_reports.log") : nil
|
|
108
|
+
@query_monitor_log_max_bytes = 10 * 1024 * 1024
|
|
109
|
+
# A developer needs the values and they are their own; anywhere else they are
|
|
110
|
+
# somebody's passwords and personal data
|
|
111
|
+
@query_monitor_redact_literals = !rails_env&.development?
|
|
91
112
|
@query_monitor_max_queries = 100
|
|
92
113
|
@query_monitor_backtrace_filter = ->(location) {
|
|
93
114
|
# Exclude gem paths, framework paths
|
|
@@ -107,6 +128,7 @@ module PgReports
|
|
|
107
128
|
ActiveModel::Type::Boolean.new.cast(env_override)
|
|
108
129
|
end
|
|
109
130
|
@raw_query_statement_timeout_ms = ENV.fetch("PG_REPORTS_RAW_QUERY_STATEMENT_TIMEOUT_MS", 5_000).to_i
|
|
131
|
+
@report_statement_timeout_ms = ENV.fetch("PG_REPORTS_REPORT_STATEMENT_TIMEOUT_MS", 30_000).to_i
|
|
110
132
|
@raw_query_rate_limit = ENV.fetch("PG_REPORTS_RAW_QUERY_RATE_LIMIT", 30).to_i
|
|
111
133
|
@raw_query_rate_limit_window_seconds = ENV.fetch("PG_REPORTS_RAW_QUERY_RATE_LIMIT_WINDOW_SECONDS", 60).to_i
|
|
112
134
|
|
|
@@ -141,6 +163,15 @@ module PgReports
|
|
|
141
163
|
PgReports.connection_registry.default_name
|
|
142
164
|
end
|
|
143
165
|
|
|
166
|
+
def query_monitor_available?
|
|
167
|
+
query_monitor_enabled && query_monitor_available_sources.any?
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Standalone has no application to subscribe to, so only sampling applies
|
|
171
|
+
def query_monitor_available_sources
|
|
172
|
+
standalone ? [:activity] : Array(query_monitor_sources).map(&:to_sym)
|
|
173
|
+
end
|
|
174
|
+
|
|
144
175
|
def telegram_configured?
|
|
145
176
|
telegram_bot_token.present? && telegram_chat_id.present?
|
|
146
177
|
end
|
|
@@ -35,6 +35,8 @@ module PgReports
|
|
|
35
35
|
when "53300" then too_many_connections(message)
|
|
36
36
|
when "42P01" then undefined_table(message)
|
|
37
37
|
when "42883" then undefined_function(message)
|
|
38
|
+
when "57014" then query_canceled(error)
|
|
39
|
+
when "55P03" then lock_wait_timed_out(message)
|
|
38
40
|
else generic(error)
|
|
39
41
|
end
|
|
40
42
|
|
|
@@ -52,6 +54,25 @@ module PgReports
|
|
|
52
54
|
nil
|
|
53
55
|
end
|
|
54
56
|
|
|
57
|
+
# 57014 also covers pg_cancel_backend, which says so in its own words
|
|
58
|
+
def query_canceled(error)
|
|
59
|
+
return generic(error) unless error.message.include?("statement timeout")
|
|
60
|
+
|
|
61
|
+
{
|
|
62
|
+
title: "Report timed out",
|
|
63
|
+
detail: I18n.t("pg_reports.ui.errors.report_timed_out", ms: PgReports.config.report_statement_timeout_ms.to_i),
|
|
64
|
+
hint: nil
|
|
65
|
+
}
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def lock_wait_timed_out(_message)
|
|
69
|
+
{
|
|
70
|
+
title: "Report blocked by a lock",
|
|
71
|
+
detail: I18n.t("pg_reports.ui.errors.report_lock_timed_out"),
|
|
72
|
+
hint: nil
|
|
73
|
+
}
|
|
74
|
+
end
|
|
75
|
+
|
|
55
76
|
def permission_denied(message)
|
|
56
77
|
target = extract_object(message, /permission denied for (?:database|schema|table|relation|view) "?([\w.]+)"?/)
|
|
57
78
|
kind = extract_object(message, /permission denied for (database|schema|table|relation|view)/)
|
|
@@ -13,6 +13,29 @@ module PgReports
|
|
|
13
13
|
class Target
|
|
14
14
|
class ConnectionFailed < PgReports::Error; end
|
|
15
15
|
|
|
16
|
+
DATABASE_SIZES_TTL = 60 # seconds
|
|
17
|
+
|
|
18
|
+
DATABASE_NAMES_SQL = <<~SQL
|
|
19
|
+
#{Executor::QUERY_TAG}
|
|
20
|
+
SELECT datname AS name
|
|
21
|
+
FROM pg_database
|
|
22
|
+
WHERE datistemplate = false AND datallowconn = true
|
|
23
|
+
ORDER BY datname
|
|
24
|
+
SQL
|
|
25
|
+
|
|
26
|
+
# pg_database_size raises for a database the role cannot connect to, which
|
|
27
|
+
# would take the whole list down over one unreadable size.
|
|
28
|
+
DATABASE_SIZES_SQL = <<~SQL
|
|
29
|
+
#{Executor::QUERY_TAG}
|
|
30
|
+
SELECT datname AS name,
|
|
31
|
+
CASE WHEN has_database_privilege(datname, 'CONNECT')
|
|
32
|
+
THEN pg_size_pretty(pg_database_size(datname))
|
|
33
|
+
END AS size
|
|
34
|
+
FROM pg_database
|
|
35
|
+
WHERE datistemplate = false AND datallowconn = true
|
|
36
|
+
ORDER BY datname
|
|
37
|
+
SQL
|
|
38
|
+
|
|
16
39
|
attr_reader :name
|
|
17
40
|
|
|
18
41
|
def initialize(name, spec)
|
|
@@ -48,17 +71,15 @@ module PgReports
|
|
|
48
71
|
|
|
49
72
|
# List all databases visible on this target's cluster (using pg_database).
|
|
50
73
|
# Result rows: { "name" => String, "size" => String, "current" => Boolean }
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
SQL
|
|
74
|
+
#
|
|
75
|
+
# pg_database_size walks every file of every database, which on a cluster with
|
|
76
|
+
# many tables takes far longer than anything else the dashboard asks for. Only
|
|
77
|
+
# a page showing the selector wants the sizes, and they change slowly, so ask
|
|
78
|
+
# for `sizes: false` to validate a name and take cached sizes otherwise.
|
|
79
|
+
def list_databases(current: nil, sizes: true)
|
|
80
|
+
rows = sizes ? databases_with_sizes : connection_for.exec_query(DATABASE_NAMES_SQL, "PgReports").to_a
|
|
59
81
|
current_db = (current || default_database).to_s
|
|
60
|
-
rows.
|
|
61
|
-
rows
|
|
82
|
+
rows.map { |r| r.merge("current" => r["name"].to_s == current_db) }
|
|
62
83
|
end
|
|
63
84
|
|
|
64
85
|
# Close all derived pools we own. The :primary AR::Base pool is never touched.
|
|
@@ -70,10 +91,22 @@ module PgReports
|
|
|
70
91
|
# Best-effort cleanup
|
|
71
92
|
end
|
|
72
93
|
@pools.clear
|
|
94
|
+
@sized_databases = nil
|
|
73
95
|
end
|
|
74
96
|
|
|
75
97
|
private
|
|
76
98
|
|
|
99
|
+
def databases_with_sizes
|
|
100
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
101
|
+
cached = @sized_databases
|
|
102
|
+
|
|
103
|
+
return cached[:rows] if cached && now - cached[:at] < DATABASE_SIZES_TTL
|
|
104
|
+
|
|
105
|
+
rows = connection_for.exec_query(DATABASE_SIZES_SQL, "PgReports").to_a
|
|
106
|
+
@sized_databases = {at: now, rows: rows}
|
|
107
|
+
rows
|
|
108
|
+
end
|
|
109
|
+
|
|
77
110
|
def normalize_spec(spec)
|
|
78
111
|
hash = case spec
|
|
79
112
|
when Hash
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PgReports
|
|
4
|
+
# Keeps database text from running as a formula when a CSV opens in a spreadsheet.
|
|
5
|
+
#
|
|
6
|
+
# A table named =HYPERLINK(...) is ordinary data here and a live formula in Excel,
|
|
7
|
+
# so anyone able to name a table or send a query could reach whoever opens the
|
|
8
|
+
# export. A leading apostrophe makes the spreadsheet read the cell as text. The
|
|
9
|
+
# exported value changes only for cells that would otherwise be evaluated.
|
|
10
|
+
module CsvCell
|
|
11
|
+
FORMULA_START = /\A\s*[=+\-@]/
|
|
12
|
+
NUMBER = /\A\s*[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?\s*\z/i
|
|
13
|
+
|
|
14
|
+
def self.safe(value)
|
|
15
|
+
return value unless value.is_a?(String) && value.match?(FORMULA_START) && !value.match?(NUMBER)
|
|
16
|
+
|
|
17
|
+
"'#{value}"
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -16,16 +16,20 @@ module PgReports
|
|
|
16
16
|
thresholds: {calls: {warning: 10000, critical: 100000}},
|
|
17
17
|
problem_fields: ["calls"]
|
|
18
18
|
},
|
|
19
|
+
frequent_queries: {
|
|
20
|
+
thresholds: {calls_per_second: {warning: 100, critical: 1000}},
|
|
21
|
+
problem_fields: ["calls_per_second"]
|
|
22
|
+
},
|
|
19
23
|
expensive_queries: {
|
|
20
24
|
thresholds: {total_time_ms: {warning: 60000, critical: 300000}},
|
|
21
25
|
problem_fields: ["total_time_ms"]
|
|
22
26
|
},
|
|
23
27
|
missing_index_queries: {
|
|
24
|
-
thresholds: {
|
|
25
|
-
problem_fields: ["
|
|
28
|
+
thresholds: {rows_per_call: {warning: 1000, critical: 10000}},
|
|
29
|
+
problem_fields: ["rows_per_call"]
|
|
26
30
|
},
|
|
27
31
|
low_cache_hit_queries: {
|
|
28
|
-
thresholds: {cache_hit_ratio: {warning:
|
|
32
|
+
thresholds: {cache_hit_ratio: {warning: 95, critical: 80, inverted: true}},
|
|
29
33
|
problem_fields: ["cache_hit_ratio"]
|
|
30
34
|
},
|
|
31
35
|
all_queries: {
|
|
@@ -51,8 +55,8 @@ module PgReports
|
|
|
51
55
|
problem_fields: []
|
|
52
56
|
},
|
|
53
57
|
missing_indexes: {
|
|
54
|
-
thresholds: {
|
|
55
|
-
problem_fields: ["
|
|
58
|
+
thresholds: {avg_seq_tup_read: {warning: 10000, critical: 100000}},
|
|
59
|
+
problem_fields: ["avg_seq_tup_read"]
|
|
56
60
|
},
|
|
57
61
|
inefficient_indexes: {
|
|
58
62
|
thresholds: {read_to_fetch_ratio: {warning: 10, critical: 100}},
|
|
@@ -62,31 +66,33 @@ module PgReports
|
|
|
62
66
|
thresholds: {},
|
|
63
67
|
problem_fields: []
|
|
64
68
|
},
|
|
69
|
+
# No limits until the estimate improves: it assumes 8 bytes per entry, so a
|
|
70
|
+
# freshly built index reads as 65-75% bloated and every row would be critical
|
|
65
71
|
bloated_indexes: {
|
|
66
|
-
thresholds: {
|
|
67
|
-
problem_fields: [
|
|
72
|
+
thresholds: {},
|
|
73
|
+
problem_fields: []
|
|
68
74
|
},
|
|
69
75
|
index_sizes: {
|
|
70
|
-
thresholds: {
|
|
71
|
-
problem_fields: ["
|
|
76
|
+
thresholds: {index_size_mb: {warning: 1024, critical: 10240}},
|
|
77
|
+
problem_fields: ["index_size_mb"]
|
|
72
78
|
},
|
|
73
79
|
fk_without_indexes: {
|
|
74
80
|
thresholds: {},
|
|
75
81
|
problem_fields: ["child_table_size_mb"]
|
|
76
82
|
},
|
|
77
83
|
index_correlation: {
|
|
78
|
-
thresholds: {
|
|
79
|
-
problem_fields: ["
|
|
84
|
+
thresholds: {abs_correlation: {warning: 0.5, critical: 0.2, inverted: true}},
|
|
85
|
+
problem_fields: ["abs_correlation"]
|
|
80
86
|
},
|
|
81
87
|
|
|
82
88
|
# === TABLES ===
|
|
83
89
|
table_sizes: {
|
|
84
|
-
thresholds: {
|
|
85
|
-
problem_fields: ["
|
|
90
|
+
thresholds: {total_size_mb: {warning: 10240, critical: 102400}},
|
|
91
|
+
problem_fields: ["total_size_mb"]
|
|
86
92
|
},
|
|
87
93
|
bloated_tables: {
|
|
88
|
-
thresholds: {
|
|
89
|
-
problem_fields: ["
|
|
94
|
+
thresholds: {bloat_percent: {warning: 10, critical: 20}},
|
|
95
|
+
problem_fields: ["bloat_percent"]
|
|
90
96
|
},
|
|
91
97
|
vacuum_needed: {
|
|
92
98
|
thresholds: {n_dead_tup: {warning: 10000, critical: 100000}},
|
|
@@ -97,7 +103,7 @@ module PgReports
|
|
|
97
103
|
problem_fields: []
|
|
98
104
|
},
|
|
99
105
|
cache_hit_ratios: {
|
|
100
|
-
thresholds: {cache_hit_ratio: {warning:
|
|
106
|
+
thresholds: {cache_hit_ratio: {warning: 95, critical: 80, inverted: true}},
|
|
101
107
|
problem_fields: ["cache_hit_ratio"]
|
|
102
108
|
},
|
|
103
109
|
seq_scans: {
|
|
@@ -115,8 +121,8 @@ module PgReports
|
|
|
115
121
|
|
|
116
122
|
# === CONNECTIONS ===
|
|
117
123
|
active_connections: {
|
|
118
|
-
thresholds: {
|
|
119
|
-
problem_fields: [
|
|
124
|
+
thresholds: {},
|
|
125
|
+
problem_fields: []
|
|
120
126
|
},
|
|
121
127
|
connection_stats: {
|
|
122
128
|
thresholds: {idle_in_transaction: {warning: 5, critical: 20}},
|
|
@@ -124,19 +130,19 @@ module PgReports
|
|
|
124
130
|
},
|
|
125
131
|
long_running_queries: {
|
|
126
132
|
thresholds: {duration_seconds: {warning: 60, critical: 300}},
|
|
127
|
-
problem_fields: ["duration_seconds"
|
|
133
|
+
problem_fields: ["duration_seconds"]
|
|
128
134
|
},
|
|
129
135
|
blocking_queries: {
|
|
130
|
-
thresholds: {
|
|
131
|
-
problem_fields: ["
|
|
136
|
+
thresholds: {blocked_duration: {warning: 10, critical: 60}},
|
|
137
|
+
problem_fields: ["blocked_duration"]
|
|
132
138
|
},
|
|
133
139
|
locks: {
|
|
134
|
-
thresholds: {
|
|
135
|
-
problem_fields: ["
|
|
140
|
+
thresholds: {waiting_seconds: {warning: 10, critical: 60}},
|
|
141
|
+
problem_fields: ["waiting_seconds"]
|
|
136
142
|
},
|
|
137
143
|
idle_connections: {
|
|
138
|
-
thresholds: {
|
|
139
|
-
problem_fields: ["
|
|
144
|
+
thresholds: {idle_in_transaction_seconds: {warning: 60, critical: 300}},
|
|
145
|
+
problem_fields: ["idle_in_transaction_seconds"]
|
|
140
146
|
},
|
|
141
147
|
pool_usage: {
|
|
142
148
|
thresholds: {utilization_pct: {warning: 70, critical: 85}},
|
|
@@ -160,8 +166,8 @@ module PgReports
|
|
|
160
166
|
|
|
161
167
|
# === SYSTEM ===
|
|
162
168
|
database_sizes: {
|
|
163
|
-
thresholds: {
|
|
164
|
-
problem_fields: ["
|
|
169
|
+
thresholds: {size_mb: {warning: 10240, critical: 102400}},
|
|
170
|
+
problem_fields: ["size_mb"]
|
|
165
171
|
},
|
|
166
172
|
settings: {
|
|
167
173
|
thresholds: {},
|
|
@@ -176,8 +182,8 @@ module PgReports
|
|
|
176
182
|
problem_fields: []
|
|
177
183
|
},
|
|
178
184
|
cache_stats: {
|
|
179
|
-
thresholds: {
|
|
180
|
-
problem_fields: ["
|
|
185
|
+
thresholds: {heap_hit_ratio: {warning: 95, critical: 90, inverted: true}},
|
|
186
|
+
problem_fields: ["heap_hit_ratio"]
|
|
181
187
|
},
|
|
182
188
|
wraparound_risk: {
|
|
183
189
|
thresholds: {pct_towards_wraparound: {warning: 50, critical: 75}},
|
|
@@ -228,7 +234,7 @@ module PgReports
|
|
|
228
234
|
},
|
|
229
235
|
unused_tables: {
|
|
230
236
|
thresholds: {total_size_mb: {warning: 10, critical: 100}},
|
|
231
|
-
problem_fields: ["
|
|
237
|
+
problem_fields: ["total_size_mb"]
|
|
232
238
|
}
|
|
233
239
|
}.freeze
|
|
234
240
|
|
|
@@ -244,6 +250,7 @@ module PgReports
|
|
|
244
250
|
reports: {
|
|
245
251
|
slow_queries: {name: "Slow Queries", description: "Queries with high mean execution time"},
|
|
246
252
|
heavy_queries: {name: "Heavy Queries", description: "Most frequently called queries"},
|
|
253
|
+
frequent_queries: {name: "Most Frequent Queries", description: "Queries executed at the highest rate (calls per second)", new: true},
|
|
247
254
|
expensive_queries: {name: "Expensive Queries", description: "Queries consuming most total time"},
|
|
248
255
|
missing_index_queries: {name: "Missing Index Queries", description: "Queries potentially missing indexes"},
|
|
249
256
|
low_cache_hit_queries: {name: "Low Cache Hit", description: "Queries with poor cache utilization"},
|
|
@@ -18,6 +18,7 @@ report:
|
|
|
18
18
|
- column_name
|
|
19
19
|
- index_name
|
|
20
20
|
- correlation
|
|
21
|
+
- abs_correlation
|
|
21
22
|
- table_size_mb
|
|
22
23
|
- idx_scan
|
|
23
24
|
|
|
@@ -28,4 +29,4 @@ report:
|
|
|
28
29
|
description: "Maximum number of results"
|
|
29
30
|
|
|
30
31
|
problem_explanations:
|
|
31
|
-
|
|
32
|
+
abs_correlation: low_correlation
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Frequent queries - queries executed at the highest rate (calls per second)
|
|
2
|
+
# Unlike heavy_queries, which ranks by the cumulative call count, this ranks by
|
|
3
|
+
# rate, so a query's position does not depend on how long it has been tracked
|
|
4
|
+
|
|
5
|
+
report:
|
|
6
|
+
name: frequent_queries
|
|
7
|
+
module: queries
|
|
8
|
+
description: "Queries executed at the highest rate"
|
|
9
|
+
|
|
10
|
+
sql:
|
|
11
|
+
category: queries
|
|
12
|
+
file: frequent_queries
|
|
13
|
+
|
|
14
|
+
title: "Most Frequent Queries (calls/sec >= ${threshold})"
|
|
15
|
+
title_vars:
|
|
16
|
+
threshold:
|
|
17
|
+
source: config
|
|
18
|
+
key: frequent_query_threshold_per_second
|
|
19
|
+
|
|
20
|
+
columns:
|
|
21
|
+
- query
|
|
22
|
+
- source
|
|
23
|
+
- calls_per_second
|
|
24
|
+
- calls
|
|
25
|
+
- mean_time_ms
|
|
26
|
+
- total_time_ms
|
|
27
|
+
- tracked_since
|
|
28
|
+
|
|
29
|
+
parameters:
|
|
30
|
+
limit:
|
|
31
|
+
type: integer
|
|
32
|
+
default: 20
|
|
33
|
+
description: "Maximum number of results"
|
|
34
|
+
|
|
35
|
+
filters:
|
|
36
|
+
- field: calls_per_second
|
|
37
|
+
operator: gte
|
|
38
|
+
value:
|
|
39
|
+
source: config
|
|
40
|
+
key: frequent_query_threshold_per_second
|
|
41
|
+
cast: float
|
|
42
|
+
|
|
43
|
+
enrichment:
|
|
44
|
+
hook: enrich_with_annotations
|
|
45
|
+
module: queries
|
|
46
|
+
|
|
47
|
+
problem_explanations:
|
|
48
|
+
calls_per_second: high_calls
|
|
49
|
+
total_time_ms: high_total_time
|
|
50
|
+
mean_time_ms: high_mean_time
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PgReports
|
|
4
|
+
# Writes the Rails migration that drops one index.
|
|
5
|
+
#
|
|
6
|
+
# Identifiers come from the database and anyone with DDL rights chooses them, so
|
|
7
|
+
# they only ever reach the generated Ruby through #inspect, which escapes quotes,
|
|
8
|
+
# newlines and `#{`. A name is never spliced in as a symbol or left bare: this
|
|
9
|
+
# file is executed by `rails db:migrate`.
|
|
10
|
+
#
|
|
11
|
+
# It drops with SQL rather than remove_index, which parses the name and, for one
|
|
12
|
+
# holding a quote or a dot, looks up something else. With if_exists that is a
|
|
13
|
+
# migration that reports success and leaves the index in place.
|
|
14
|
+
class DropIndexMigration
|
|
15
|
+
# @param index [Hash] a pg_indexes row: schemaname, indexname, indexdef
|
|
16
|
+
# @param version [#to_s] the Migration[] version, from the host's Rails
|
|
17
|
+
def initialize(index, version:)
|
|
18
|
+
@schema = index.fetch("schemaname")
|
|
19
|
+
@index = index.fetch("indexname")
|
|
20
|
+
@definition = index.fetch("indexdef")
|
|
21
|
+
@version = version
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Which migrations this would clash with: Rails refuses two of the same name.
|
|
25
|
+
def name
|
|
26
|
+
slug = @index.downcase.gsub(/[^a-z0-9]+/, "_").gsub(/\A_+|_+\z/, "")
|
|
27
|
+
"remove_#{slug.empty? ? "index" : slug}"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def file_name(timestamp)
|
|
31
|
+
"#{timestamp}_#{name}.rb"
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def code
|
|
35
|
+
<<~RUBY
|
|
36
|
+
# frozen_string_literal: true
|
|
37
|
+
|
|
38
|
+
class #{name.camelize} < ActiveRecord::Migration[#{@version}]
|
|
39
|
+
def up
|
|
40
|
+
execute #{drop_statement.inspect}
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def down
|
|
44
|
+
execute #{@definition.inspect}
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
RUBY
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def drop_statement
|
|
53
|
+
"DROP INDEX IF EXISTS #{PG::Connection.quote_ident([@schema, @index])}"
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
data/lib/pg_reports/engine.rb
CHANGED
|
@@ -13,9 +13,10 @@ module PgReports
|
|
|
13
13
|
config.i18n.load_path += Dir[root.join("config", "locales", "*.yml")]
|
|
14
14
|
end
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
# after_initialize so that the host's initializers have set the configuration
|
|
17
|
+
config.after_initialize do
|
|
18
|
+
if PgReports.config.query_monitor_enabled && PgReports.config.query_monitor_available_sources.include?(:notifications)
|
|
19
|
+
PgReports::QueryMonitor.instance.subscribe
|
|
19
20
|
end
|
|
20
21
|
end
|
|
21
22
|
|