pg_reports 0.9.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +82 -0
- data/README.md +12 -8
- data/app/controllers/pg_reports/dashboard_controller.rb +277 -221
- data/app/controllers/pg_reports/metrics_controller.rb +13 -4
- data/app/views/layouts/pg_reports/application.html.erb +222 -93
- data/app/views/pg_reports/dashboard/_settings_modal.html.erb +55 -0
- data/app/views/pg_reports/dashboard/_show_modals.html.erb +1 -42
- data/app/views/pg_reports/dashboard/_show_scripts.html.erb +75 -105
- data/app/views/pg_reports/dashboard/index.html.erb +346 -238
- data/app/views/pg_reports/dashboard/show.html.erb +1 -1
- data/bin/pg_reports +2 -2
- data/config/brakeman.ignore +10 -56
- data/config/locales/en.yml +45 -11
- data/config/locales/ru.yml +44 -11
- data/config/locales/uk.yml +44 -11
- data/config/routes.rb +2 -0
- data/lib/pg_reports/access_policy.rb +39 -0
- data/lib/pg_reports/compatibility.rb +10 -2
- data/lib/pg_reports/configuration.rb +33 -2
- data/lib/pg_reports/connection/error_translator.rb +21 -0
- data/lib/pg_reports/connection/target.rb +43 -10
- data/lib/pg_reports/csv_cell.rb +20 -0
- data/lib/pg_reports/dashboard/reports_registry.rb +37 -30
- data/lib/pg_reports/definitions/connections/connection_stats.yml +4 -0
- data/lib/pg_reports/definitions/connections/idle_connections.yml +4 -0
- data/lib/pg_reports/definitions/connections/locks.yml +1 -0
- data/lib/pg_reports/definitions/connections/long_running_queries.yml +0 -1
- data/lib/pg_reports/definitions/indexes/bloated_indexes.yml +1 -2
- data/lib/pg_reports/definitions/indexes/index_correlation.yml +2 -1
- data/lib/pg_reports/definitions/indexes/missing_indexes.yml +1 -0
- data/lib/pg_reports/definitions/queries/frequent_queries.yml +50 -0
- data/lib/pg_reports/definitions/system/cache_stats.yml +2 -1
- data/lib/pg_reports/definitions/tables/bloated_tables.yml +1 -2
- data/lib/pg_reports/definitions/tables/recently_modified.yml +3 -3
- data/lib/pg_reports/definitions/tables/unused_tables.yml +1 -1
- data/lib/pg_reports/drop_index_migration.rb +56 -0
- data/lib/pg_reports/engine.rb +4 -3
- data/lib/pg_reports/executor.rb +46 -9
- data/lib/pg_reports/modules/queries.rb +1 -0
- data/lib/pg_reports/query_monitor.rb +478 -192
- data/lib/pg_reports/rate_limiter.rb +63 -0
- data/lib/pg_reports/report.rb +1 -1
- data/lib/pg_reports/report_definition.rb +11 -1
- data/lib/pg_reports/sql/connections/connection_stats.sql +2 -1
- data/lib/pg_reports/sql/connections/idle_connections.sql +1 -0
- data/lib/pg_reports/sql/connections/locks.sql +1 -0
- data/lib/pg_reports/sql/indexes/bloated_indexes.sql +16 -8
- data/lib/pg_reports/sql/queries/activity_sample.sql +22 -0
- data/lib/pg_reports/sql/queries/frequent_queries.sql +42 -0
- data/lib/pg_reports/sql_redactor.rb +32 -0
- data/lib/pg_reports/standalone.rb +83 -20
- data/lib/pg_reports/version.rb +1 -1
- data/lib/pg_reports.rb +7 -2
- metadata +18 -9
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "singleton"
|
|
4
|
+
require "active_support/cache"
|
|
5
|
+
|
|
6
|
+
module PgReports
|
|
7
|
+
# Counts requests per key in fixed windows.
|
|
8
|
+
#
|
|
9
|
+
# The window's number is part of the cache key, so a refused request cannot push
|
|
10
|
+
# the end of the window back and a client recovers when the window turns over.
|
|
11
|
+
# The price of a fixed window is that a client can spend two windows' allowance
|
|
12
|
+
# across the boundary, which is fine for a guard against hammering.
|
|
13
|
+
#
|
|
14
|
+
# Counting uses the cache's atomic increment. That is exact on Redis, Memcached
|
|
15
|
+
# and MemoryStore. FileStore's increment is a read and a write: under load from
|
|
16
|
+
# several processes it loses updates and this counts low.
|
|
17
|
+
#
|
|
18
|
+
# When the cache cannot count, because it is a NullStore or it is failing, each
|
|
19
|
+
# process counts for itself. A throttle that switches off whenever Redis does
|
|
20
|
+
# would be gone exactly when the database needs it.
|
|
21
|
+
class RateLimiter
|
|
22
|
+
include Singleton
|
|
23
|
+
|
|
24
|
+
KEY_PREFIX = "pg_reports:rate_limit"
|
|
25
|
+
|
|
26
|
+
# @return [Integer, nil] seconds until the window turns over if this request is
|
|
27
|
+
# over the limit, nil if it may proceed
|
|
28
|
+
def retry_after(key, limit:, window:, now: Time.now.to_i)
|
|
29
|
+
window = [window.to_i, 1].max
|
|
30
|
+
slot, elapsed = now.divmod(window)
|
|
31
|
+
|
|
32
|
+
return if count("#{KEY_PREFIX}:#{key}:#{slot}", window) <= limit
|
|
33
|
+
|
|
34
|
+
window - elapsed
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def count(key, window)
|
|
40
|
+
increment(shared_store, key, window) || increment(local_store, key, window)
|
|
41
|
+
rescue StandardError, NotImplementedError => e
|
|
42
|
+
Rails.logger&.warn("PgReports: Rate limit cache unavailable: #{e.message}")
|
|
43
|
+
increment(local_store, key, window)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# The write creates the raw counter, which is the form Redis and Memcached can
|
|
47
|
+
# increment, and does nothing if another request already has.
|
|
48
|
+
def increment(store, key, window)
|
|
49
|
+
return unless store
|
|
50
|
+
|
|
51
|
+
store.write(key, 0, raw: true, unless_exist: true, expires_in: window)
|
|
52
|
+
store.increment(key, 1, raw: true, expires_in: window)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def shared_store
|
|
56
|
+
Rails.cache if Rails.respond_to?(:cache)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def local_store
|
|
60
|
+
@local_store ||= ActiveSupport::Cache::MemoryStore.new
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
data/lib/pg_reports/report.rb
CHANGED
|
@@ -57,8 +57,18 @@ module PgReports
|
|
|
57
57
|
def apply_filters(data, params)
|
|
58
58
|
return data unless config["filters"]
|
|
59
59
|
|
|
60
|
+
values = parameter_defaults.merge(params.compact)
|
|
61
|
+
|
|
60
62
|
config["filters"].reduce(data) do |filtered, filter_config|
|
|
61
|
-
Filter.new(filter_config).apply(filtered,
|
|
63
|
+
Filter.new(filter_config).apply(filtered, values)
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# The dashboard's form sends these, but /metrics, downloads, Telegram and direct
|
|
68
|
+
# calls send nothing and must get the same report.
|
|
69
|
+
def parameter_defaults
|
|
70
|
+
(config["parameters"] || {}).each_with_object({}) do |(name, parameter), defaults|
|
|
71
|
+
defaults[name.to_sym] = parameter["default"]
|
|
62
72
|
end
|
|
63
73
|
end
|
|
64
74
|
|
|
@@ -5,7 +5,8 @@ SELECT
|
|
|
5
5
|
datname AS database,
|
|
6
6
|
state,
|
|
7
7
|
COUNT(*) AS count,
|
|
8
|
-
COUNT(*) FILTER (WHERE wait_event IS NOT NULL) AS waiting_count
|
|
8
|
+
COUNT(*) FILTER (WHERE wait_event IS NOT NULL) AS waiting_count,
|
|
9
|
+
COUNT(*) FILTER (WHERE state LIKE 'idle in transaction%') AS idle_in_transaction
|
|
9
10
|
FROM pg_stat_activity
|
|
10
11
|
WHERE datname IS NOT NULL
|
|
11
12
|
AND pid != pg_backend_pid()
|
|
@@ -9,6 +9,7 @@ SELECT
|
|
|
9
9
|
client_addr,
|
|
10
10
|
state,
|
|
11
11
|
EXTRACT(EPOCH FROM (NOW() - state_change)) AS idle_duration,
|
|
12
|
+
CASE WHEN state LIKE 'idle in transaction%' THEN EXTRACT(EPOCH FROM (NOW() - state_change)) END AS idle_in_transaction_seconds,
|
|
12
13
|
state_change,
|
|
13
14
|
query_start,
|
|
14
15
|
LEFT(query, 200) AS last_query
|
|
@@ -11,6 +11,7 @@ SELECT
|
|
|
11
11
|
l.granted,
|
|
12
12
|
NOT l.granted AS waiting,
|
|
13
13
|
EXTRACT(EPOCH FROM (NOW() - a.query_start)) AS query_duration,
|
|
14
|
+
CASE WHEN NOT l.granted THEN EXTRACT(EPOCH FROM (NOW() - a.query_start)) END AS waiting_seconds,
|
|
14
15
|
LEFT(a.query, 300) AS query
|
|
15
16
|
FROM pg_locks l
|
|
16
17
|
JOIN pg_stat_activity a ON l.pid = a.pid
|
|
@@ -1,27 +1,35 @@
|
|
|
1
1
|
-- Bloated indexes: indexes with estimated bloat
|
|
2
2
|
-- Uses pgstattuple extension if available, otherwise estimates
|
|
3
3
|
|
|
4
|
-
WITH
|
|
4
|
+
WITH named_indexes AS (
|
|
5
|
+
SELECT
|
|
6
|
+
schemaname,
|
|
7
|
+
tablename,
|
|
8
|
+
indexname,
|
|
9
|
+
format('%I.%I', schemaname, indexname)::regclass AS index_oid
|
|
10
|
+
FROM pg_indexes
|
|
11
|
+
WHERE schemaname NOT IN ('pg_catalog', 'information_schema')
|
|
12
|
+
),
|
|
13
|
+
index_stats AS (
|
|
5
14
|
SELECT
|
|
6
15
|
schemaname AS schema,
|
|
7
16
|
tablename AS table_name,
|
|
8
17
|
indexname AS index_name,
|
|
9
|
-
pg_relation_size(
|
|
18
|
+
pg_relation_size(index_oid) AS index_size,
|
|
10
19
|
-- Estimate bloat using relation page count vs expected
|
|
11
20
|
CASE
|
|
12
|
-
WHEN pg_relation_size(
|
|
21
|
+
WHEN pg_relation_size(index_oid) > 0 THEN
|
|
13
22
|
ROUND(
|
|
14
23
|
(
|
|
15
|
-
(pg_relation_size(
|
|
16
|
-
(SELECT reltuples * 8 FROM pg_class WHERE
|
|
17
|
-
* 100.0 / pg_relation_size(
|
|
24
|
+
(pg_relation_size(index_oid) -
|
|
25
|
+
(SELECT reltuples * 8 FROM pg_class WHERE oid = index_oid))
|
|
26
|
+
* 100.0 / pg_relation_size(index_oid)
|
|
18
27
|
)::numeric,
|
|
19
28
|
2
|
|
20
29
|
)
|
|
21
30
|
ELSE 0
|
|
22
31
|
END AS estimated_bloat_percent
|
|
23
|
-
FROM
|
|
24
|
-
WHERE schemaname NOT IN ('pg_catalog', 'information_schema')
|
|
32
|
+
FROM named_indexes
|
|
25
33
|
)
|
|
26
34
|
SELECT
|
|
27
35
|
schema,
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
-- Query Monitor sample: the statement each backend finished most recently
|
|
2
|
+
-- A backend that is no longer active keeps its last statement in `query`, with
|
|
3
|
+
-- its start in query_start and its end in state_change.
|
|
4
|
+
--
|
|
5
|
+
-- Without pg_read_all_stats, another role's rows arrive with a NULL state and
|
|
6
|
+
-- NULL backend_type, so those conditions must let them through: the monitor
|
|
7
|
+
-- counts them to tell the viewer what it cannot see.
|
|
8
|
+
|
|
9
|
+
SELECT
|
|
10
|
+
pid,
|
|
11
|
+
application_name,
|
|
12
|
+
query,
|
|
13
|
+
query_start,
|
|
14
|
+
ROUND((EXTRACT(EPOCH FROM (state_change - query_start)) * 1000)::numeric, 2) AS duration_ms
|
|
15
|
+
FROM pg_stat_activity
|
|
16
|
+
WHERE datname = current_database()
|
|
17
|
+
AND pid <> pg_backend_pid()
|
|
18
|
+
AND COALESCE(backend_type, 'client backend') = 'client backend'
|
|
19
|
+
AND application_name IS DISTINCT FROM 'pg_reports'
|
|
20
|
+
AND query NOT LIKE '%/* pg_reports */%'
|
|
21
|
+
AND state IS DISTINCT FROM 'active'
|
|
22
|
+
ORDER BY query_start;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
-- Frequent queries: queries executed at the highest rate (calls per second)
|
|
2
|
+
-- Requires pg_stat_statements 1.9+ (PostgreSQL 14+) for pg_stat_statements_info
|
|
3
|
+
--
|
|
4
|
+
-- pg_stat_statements only keeps cumulative counters, so the rate is an average
|
|
5
|
+
-- over the time each entry has been tracked: since the entry was created
|
|
6
|
+
-- (stats_since, PostgreSQL 17+) or since the last full reset on older versions.
|
|
7
|
+
-- stats_since is read through to_jsonb so that one statement runs on both;
|
|
8
|
+
-- GREATEST ignores the NULL it yields where the column does not exist.
|
|
9
|
+
|
|
10
|
+
WITH tracked AS (
|
|
11
|
+
SELECT
|
|
12
|
+
s.query,
|
|
13
|
+
s.calls,
|
|
14
|
+
s.total_exec_time,
|
|
15
|
+
s.mean_exec_time,
|
|
16
|
+
GREATEST((to_jsonb(s) ->> 'stats_since')::timestamptz, i.stats_reset) AS tracked_since
|
|
17
|
+
FROM pg_stat_statements s
|
|
18
|
+
JOIN pg_database d ON s.dbid = d.oid
|
|
19
|
+
CROSS JOIN pg_stat_statements_info i
|
|
20
|
+
WHERE s.calls > 0
|
|
21
|
+
AND s.query NOT LIKE '%pg_stat_statements%'
|
|
22
|
+
AND s.query NOT LIKE 'COMMIT%'
|
|
23
|
+
AND s.query NOT LIKE 'BEGIN%'
|
|
24
|
+
AND d.datname = current_database()
|
|
25
|
+
),
|
|
26
|
+
rated AS (
|
|
27
|
+
SELECT
|
|
28
|
+
*,
|
|
29
|
+
-- The 60-second floor keeps entries seen moments ago from reporting a huge rate
|
|
30
|
+
calls / GREATEST(EXTRACT(EPOCH FROM now() - tracked_since), 60) AS rate
|
|
31
|
+
FROM tracked
|
|
32
|
+
)
|
|
33
|
+
SELECT
|
|
34
|
+
query,
|
|
35
|
+
ROUND(rate::numeric, 2) AS calls_per_second,
|
|
36
|
+
calls,
|
|
37
|
+
ROUND(mean_exec_time::numeric, 2) AS mean_time_ms,
|
|
38
|
+
ROUND(total_exec_time::numeric, 2) AS total_time_ms,
|
|
39
|
+
tracked_since
|
|
40
|
+
FROM rated
|
|
41
|
+
ORDER BY rate DESC
|
|
42
|
+
LIMIT 100;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PgReports
|
|
4
|
+
# Hides the string literals in a SQL statement, which is where passwords, tokens
|
|
5
|
+
# and personal data sit in captured queries.
|
|
6
|
+
#
|
|
7
|
+
# Numbers stay: they are what makes a captured query readable, and they rarely
|
|
8
|
+
# identify more than a row. Quoted identifiers and comments are matched only so
|
|
9
|
+
# that a quote inside them is not taken for the start of a string.
|
|
10
|
+
module SqlRedactor
|
|
11
|
+
PLACEHOLDER = "'?'"
|
|
12
|
+
|
|
13
|
+
# Every alternative ends at \z as well as at its closing quote. pg_stat_activity
|
|
14
|
+
# and the monitor's own limit both cut statements short, so the literal holding
|
|
15
|
+
# the secret can be the one with no end. The inner repeats are possessive: their
|
|
16
|
+
# alternatives cannot overlap, and a megabyte of text must not backtrack.
|
|
17
|
+
TOKEN = /
|
|
18
|
+
"(?:[^"]|"")*+(?:"|\z) # identifier
|
|
19
|
+
| --[^\n]*+ # line comment
|
|
20
|
+
| \/\*.*?(?:\*\/|\z) # block comment
|
|
21
|
+
| (?<literal>
|
|
22
|
+
(?<![\w$])[eE]'(?:[^'\\]|''|\\.)*+(?:'|\z) # backslash escapes
|
|
23
|
+
| '(?:[^']|'')*+(?:'|\z) # standard string
|
|
24
|
+
| \$(?<tag>[A-Za-z_]\w*|)\$.*?(?:\$\k<tag>\$|\z) # dollar-quoted
|
|
25
|
+
)
|
|
26
|
+
/mx
|
|
27
|
+
|
|
28
|
+
def self.redact(sql)
|
|
29
|
+
sql.gsub(TOKEN) { $~[:literal] ? PLACEHOLDER : $~[0] }
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "ipaddr"
|
|
3
4
|
require "securerandom"
|
|
4
5
|
|
|
5
6
|
module PgReports
|
|
@@ -20,13 +21,15 @@ module PgReports
|
|
|
20
21
|
DEFAULT_PORT = 4000
|
|
21
22
|
DEFAULT_HOST = "127.0.0.1"
|
|
22
23
|
DEFAULT_MOUNT = "/"
|
|
24
|
+
LOOPBACK_HOSTS = %w[127.0.0.1 localhost ::1].freeze
|
|
23
25
|
|
|
24
26
|
# Rack handlers tried, in order, when none is named explicitly.
|
|
25
27
|
CANDIDATE_SERVERS = %w[puma webrick].freeze
|
|
26
28
|
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
|
|
29
|
+
# Where a config file tends to live, relative to the working directory. Named
|
|
30
|
+
# only to say that one was skipped: a config file is Ruby, and starting this
|
|
31
|
+
# tool inside a cloned repository must not run what that repository ships.
|
|
32
|
+
CONVENTIONAL_CONFIG_FILES = %w[pg_reports.rb config/pg_reports.rb].freeze
|
|
30
33
|
|
|
31
34
|
class ServerUnavailable < PgReports::Error; end
|
|
32
35
|
|
|
@@ -39,7 +42,7 @@ module PgReports
|
|
|
39
42
|
# from DATABASE_URL or libpq-style PG* env vars
|
|
40
43
|
# @param server [String, nil] Rack handler name to force (e.g. "puma")
|
|
41
44
|
# @param config_file [String, nil] path to a Ruby config file that calls
|
|
42
|
-
# `PgReports.configure
|
|
45
|
+
# `PgReports.configure`
|
|
43
46
|
# @param overrides [Hash] per-setting overrides applied last (from CLI flags);
|
|
44
47
|
# nil values are ignored. Keys are Configuration attribute names.
|
|
45
48
|
def run(port: DEFAULT_PORT, host: DEFAULT_HOST, mount_path: DEFAULT_MOUNT,
|
|
@@ -50,11 +53,22 @@ module PgReports
|
|
|
50
53
|
# and database switching / multi-cluster all work unchanged.
|
|
51
54
|
ENV["DATABASE_URL"] = connection_url(database_url)
|
|
52
55
|
|
|
56
|
+
# libpq reads this for every connection, including the per-database pools
|
|
57
|
+
# opened later, which is what lets the Query Monitor leave them out.
|
|
58
|
+
ENV["PGAPPNAME"] = QueryMonitor::APPLICATION_NAME
|
|
59
|
+
|
|
53
60
|
# Mark this process as standalone so the dashboard can hide reports that
|
|
54
61
|
# only make sense with a host app (e.g. Schema Analysis, which introspects
|
|
55
62
|
# the host application's ActiveRecord models — there are none here).
|
|
56
63
|
PgReports.config.standalone = true
|
|
57
64
|
|
|
65
|
+
# The production default protects a host application's queries, and there
|
|
66
|
+
# is no host application here. A config file can still turn it off below.
|
|
67
|
+
PgReports.config.query_monitor_enabled = true
|
|
68
|
+
|
|
69
|
+
# It samples every client of the database, not one developer's own application
|
|
70
|
+
PgReports.config.query_monitor_redact_literals = true
|
|
71
|
+
|
|
58
72
|
# Layer settings on top of the ENV-derived defaults: config file first, then
|
|
59
73
|
# explicit CLI overrides win.
|
|
60
74
|
apply_configuration(config_file: config_file, overrides: overrides)
|
|
@@ -88,7 +102,38 @@ module PgReports
|
|
|
88
102
|
userinfo << "@"
|
|
89
103
|
end
|
|
90
104
|
|
|
91
|
-
"
|
|
105
|
+
path = "/#{ERB::Util.url_encode(database)}"
|
|
106
|
+
|
|
107
|
+
# A directory is how libpq spells a Unix socket. It cannot be the URL's
|
|
108
|
+
# authority, where the first slash would start the path, and URI refuses
|
|
109
|
+
# credentials without a host, so everything moves to the query.
|
|
110
|
+
if host.start_with?("/")
|
|
111
|
+
query = {user: user, password: password, host: host, port: port}
|
|
112
|
+
.reject { |_key, value| value.to_s.empty? }
|
|
113
|
+
.map { |key, value| "#{key}=#{ERB::Util.url_encode(value)}" }
|
|
114
|
+
return "postgresql://#{path}?#{query.join("&")}"
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
authority = host.include?(":") ? "[#{host}]" : host
|
|
118
|
+
"postgresql://#{userinfo}#{authority}:#{port}#{path}"
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# For anything shown to a person: the URL can carry the password, in either of
|
|
122
|
+
# the two places #connection_url puts it.
|
|
123
|
+
def redact_url(url)
|
|
124
|
+
url.to_s.sub(%r{(//[^:/@]*:).*@}, '\1***@').sub(/([?&]password=)[^&]*/, '\1***')
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# What Host headers to answer to. Without credentials the only thing keeping
|
|
128
|
+
# the dashboard private is that a request comes from this machine, and a web
|
|
129
|
+
# page can arrange that: it points its own domain at 127.0.0.1 (DNS rebinding),
|
|
130
|
+
# and the browser then sends that page's requests here as same-origin. Such a
|
|
131
|
+
# request still names the attacker's domain in Host, so it is refused here.
|
|
132
|
+
# Credentials make the check pointless, and the real name unknowable.
|
|
133
|
+
def allowed_hosts
|
|
134
|
+
return [] if PgReports.config.dashboard_auth.present?
|
|
135
|
+
|
|
136
|
+
["localhost", ".localhost", IPAddr.new("127.0.0.0/8"), IPAddr.new("::1")]
|
|
92
137
|
end
|
|
93
138
|
|
|
94
139
|
private
|
|
@@ -102,8 +147,11 @@ module PgReports
|
|
|
102
147
|
# flag — thresholds, Telegram, Grafana favorites, even the dashboard_auth
|
|
103
148
|
# proc. CLI flags cover only the handful of common security toggles.
|
|
104
149
|
def apply_configuration(config_file:, overrides:)
|
|
105
|
-
|
|
106
|
-
|
|
150
|
+
if config_file
|
|
151
|
+
load_config_file(config_file)
|
|
152
|
+
else
|
|
153
|
+
note_skipped_config_file
|
|
154
|
+
end
|
|
107
155
|
|
|
108
156
|
overrides.each do |key, value|
|
|
109
157
|
next if value.nil? # unset flag — leave the file/ENV value in place
|
|
@@ -111,12 +159,9 @@ module PgReports
|
|
|
111
159
|
end
|
|
112
160
|
end
|
|
113
161
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
DEFAULT_CONFIG_FILES
|
|
118
|
-
.map { |f| File.expand_path(f, Dir.pwd) }
|
|
119
|
-
.find { |f| File.file?(f) }
|
|
162
|
+
def note_skipped_config_file
|
|
163
|
+
found = CONVENTIONAL_CONFIG_FILES.find { |file| File.file?(File.expand_path(file, Dir.pwd)) }
|
|
164
|
+
warn "pg_reports: not loading #{found}, pass --config #{found} to use it" if found
|
|
120
165
|
end
|
|
121
166
|
|
|
122
167
|
# Evaluate a Ruby config file. A missing explicit path is an error (the user
|
|
@@ -148,6 +193,8 @@ module PgReports
|
|
|
148
193
|
require "pg_reports/engine"
|
|
149
194
|
|
|
150
195
|
target_mount = mount_path
|
|
196
|
+
hosts = allowed_hosts
|
|
197
|
+
token_protected_metrics = PgReports.config.grafana_metrics_token.present?
|
|
151
198
|
# A throwaway, empty app root. We must NOT use the gem root here — Rails
|
|
152
199
|
# would then load the gem's engine config/routes.rb (and config/locales) as
|
|
153
200
|
# the *application's* own, which double-loads and breaks. The engine loads
|
|
@@ -158,10 +205,15 @@ module PgReports
|
|
|
158
205
|
Class.new(Rails::Application) do
|
|
159
206
|
config.root = app_root
|
|
160
207
|
config.eager_load = false
|
|
161
|
-
config.consider_all_requests_local =
|
|
208
|
+
config.consider_all_requests_local = false
|
|
162
209
|
config.secret_key_base = ENV["SECRET_KEY_BASE"] || SecureRandom.hex(64)
|
|
163
210
|
config.session_store :cookie_store, key: "_pg_reports_session"
|
|
164
|
-
config.hosts
|
|
211
|
+
config.hosts = hosts
|
|
212
|
+
# A scraper asks for the server by whatever name it has, and the bearer
|
|
213
|
+
# token is what protects this endpoint
|
|
214
|
+
config.host_authorization = {exclude: ->(request) { request.path.end_with?("/metrics") }} if token_protected_metrics
|
|
215
|
+
# Parameters reach the log at the default level, and SQL carries data
|
|
216
|
+
config.filter_parameters += %i[query params]
|
|
165
217
|
config.logger = ::Logger.new($stdout)
|
|
166
218
|
config.log_level = (ENV["LOG_LEVEL"] || "info").to_sym
|
|
167
219
|
config.active_support.report_deprecations = false
|
|
@@ -175,20 +227,19 @@ module PgReports
|
|
|
175
227
|
# Force an actual connection so the user gets a clear error at startup rather
|
|
176
228
|
# than a 500 on the first request when the database is unreachable.
|
|
177
229
|
def verify_connection!
|
|
178
|
-
|
|
230
|
+
# Rails 7.2+ connects lazily, so only a query proves anything
|
|
231
|
+
ActiveRecord::Base.connection.execute("SELECT 1")
|
|
179
232
|
rescue => e
|
|
180
|
-
raise PgReports::Error, "Cannot connect to the database (#{ENV["DATABASE_URL"]}): #{e.message}"
|
|
233
|
+
raise PgReports::Error, "Cannot connect to the database (#{redact_url(ENV["DATABASE_URL"])}): #{e.message}"
|
|
181
234
|
end
|
|
182
235
|
|
|
183
236
|
# Find a usable Rack handler. Honors an explicit name, otherwise tries the
|
|
184
237
|
# candidates in order and uses the first that is installed.
|
|
185
238
|
def resolve_server(name)
|
|
186
|
-
require "rackup"
|
|
187
|
-
|
|
188
239
|
candidates = name ? [name] : CANDIDATE_SERVERS
|
|
189
240
|
candidates.each do |candidate|
|
|
190
241
|
handler = begin
|
|
191
|
-
|
|
242
|
+
rack_handlers.get(candidate)
|
|
192
243
|
rescue LoadError, NameError
|
|
193
244
|
nil
|
|
194
245
|
end
|
|
@@ -202,9 +253,21 @@ module PgReports
|
|
|
202
253
|
MSG
|
|
203
254
|
end
|
|
204
255
|
|
|
256
|
+
# Rackup split from Rack with Rack 3, so Rails before 7.1 does not bring it
|
|
257
|
+
def rack_handlers
|
|
258
|
+
require "rackup"
|
|
259
|
+
Rackup::Handler
|
|
260
|
+
rescue LoadError
|
|
261
|
+
require "rack"
|
|
262
|
+
Rack::Handler
|
|
263
|
+
end
|
|
264
|
+
|
|
205
265
|
def banner(host:, port:, server:)
|
|
206
266
|
url_host = (host == "0.0.0.0") ? "localhost" : host
|
|
207
267
|
warn "pg_reports: serving dashboard via #{server} on http://#{url_host}:#{port} (Ctrl-C to stop)"
|
|
268
|
+
return if PgReports.config.dashboard_auth.present? || LOOPBACK_HOSTS.include?(host)
|
|
269
|
+
|
|
270
|
+
warn "pg_reports: no dashboard_auth is set, so requests from other machines get a 403 (see docs/standalone.md)"
|
|
208
271
|
end
|
|
209
272
|
end
|
|
210
273
|
end
|
data/lib/pg_reports/version.rb
CHANGED
data/lib/pg_reports.rb
CHANGED
|
@@ -11,17 +11,22 @@ require "active_record"
|
|
|
11
11
|
|
|
12
12
|
require_relative "pg_reports/version"
|
|
13
13
|
require_relative "pg_reports/error"
|
|
14
|
-
require_relative "pg_reports/connection/target"
|
|
15
14
|
require_relative "pg_reports/connection/registry"
|
|
16
15
|
require_relative "pg_reports/connection/error_translator"
|
|
17
16
|
require_relative "pg_reports/compatibility"
|
|
18
17
|
require_relative "pg_reports/configuration"
|
|
18
|
+
require_relative "pg_reports/access_policy"
|
|
19
|
+
require_relative "pg_reports/rate_limiter"
|
|
19
20
|
require_relative "pg_reports/sql_loader"
|
|
20
21
|
require_relative "pg_reports/executor"
|
|
22
|
+
require_relative "pg_reports/connection/target"
|
|
23
|
+
require_relative "pg_reports/csv_cell"
|
|
21
24
|
require_relative "pg_reports/report"
|
|
22
25
|
require_relative "pg_reports/telegram_sender"
|
|
23
26
|
require_relative "pg_reports/annotation_parser"
|
|
27
|
+
require_relative "pg_reports/sql_redactor"
|
|
24
28
|
require_relative "pg_reports/explain_analyzer"
|
|
29
|
+
require_relative "pg_reports/drop_index_migration"
|
|
25
30
|
require_relative "pg_reports/query_monitor"
|
|
26
31
|
|
|
27
32
|
# YAML-based report system
|
|
@@ -55,7 +60,7 @@ require_relative "pg_reports/standalone"
|
|
|
55
60
|
module PgReports
|
|
56
61
|
class << self
|
|
57
62
|
# Query analysis methods
|
|
58
|
-
delegate :slow_queries, :heavy_queries, :expensive_queries,
|
|
63
|
+
delegate :slow_queries, :heavy_queries, :frequent_queries, :expensive_queries,
|
|
59
64
|
:missing_index_queries, :low_cache_hit_queries, :all_queries,
|
|
60
65
|
:reset_statistics!, to: Modules::Queries
|
|
61
66
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pg_reports
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 1.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Eldar Avatov
|
|
@@ -15,56 +15,56 @@ dependencies:
|
|
|
15
15
|
requirements:
|
|
16
16
|
- - ">="
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
|
-
version: '
|
|
18
|
+
version: '6.1'
|
|
19
19
|
type: :runtime
|
|
20
20
|
prerelease: false
|
|
21
21
|
version_requirements: !ruby/object:Gem::Requirement
|
|
22
22
|
requirements:
|
|
23
23
|
- - ">="
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
|
-
version: '
|
|
25
|
+
version: '6.1'
|
|
26
26
|
- !ruby/object:Gem::Dependency
|
|
27
27
|
name: activerecord
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
|
29
29
|
requirements:
|
|
30
30
|
- - ">="
|
|
31
31
|
- !ruby/object:Gem::Version
|
|
32
|
-
version: '
|
|
32
|
+
version: '6.1'
|
|
33
33
|
type: :runtime
|
|
34
34
|
prerelease: false
|
|
35
35
|
version_requirements: !ruby/object:Gem::Requirement
|
|
36
36
|
requirements:
|
|
37
37
|
- - ">="
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
|
-
version: '
|
|
39
|
+
version: '6.1'
|
|
40
40
|
- !ruby/object:Gem::Dependency
|
|
41
41
|
name: actionpack
|
|
42
42
|
requirement: !ruby/object:Gem::Requirement
|
|
43
43
|
requirements:
|
|
44
44
|
- - ">="
|
|
45
45
|
- !ruby/object:Gem::Version
|
|
46
|
-
version: '
|
|
46
|
+
version: '6.1'
|
|
47
47
|
type: :runtime
|
|
48
48
|
prerelease: false
|
|
49
49
|
version_requirements: !ruby/object:Gem::Requirement
|
|
50
50
|
requirements:
|
|
51
51
|
- - ">="
|
|
52
52
|
- !ruby/object:Gem::Version
|
|
53
|
-
version: '
|
|
53
|
+
version: '6.1'
|
|
54
54
|
- !ruby/object:Gem::Dependency
|
|
55
55
|
name: railties
|
|
56
56
|
requirement: !ruby/object:Gem::Requirement
|
|
57
57
|
requirements:
|
|
58
58
|
- - ">="
|
|
59
59
|
- !ruby/object:Gem::Version
|
|
60
|
-
version: '
|
|
60
|
+
version: '6.1'
|
|
61
61
|
type: :runtime
|
|
62
62
|
prerelease: false
|
|
63
63
|
version_requirements: !ruby/object:Gem::Requirement
|
|
64
64
|
requirements:
|
|
65
65
|
- - ">="
|
|
66
66
|
- !ruby/object:Gem::Version
|
|
67
|
-
version: '
|
|
67
|
+
version: '6.1'
|
|
68
68
|
- !ruby/object:Gem::Dependency
|
|
69
69
|
name: pg
|
|
70
70
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -153,6 +153,7 @@ files:
|
|
|
153
153
|
- app/views/layouts/pg_reports/application.html.erb
|
|
154
154
|
- app/views/pg_reports/dashboard/_database_selector.html.erb
|
|
155
155
|
- app/views/pg_reports/dashboard/_fake_source_data.html.erb
|
|
156
|
+
- app/views/pg_reports/dashboard/_settings_modal.html.erb
|
|
156
157
|
- app/views/pg_reports/dashboard/_show_modals.html.erb
|
|
157
158
|
- app/views/pg_reports/dashboard/_show_scripts.html.erb
|
|
158
159
|
- app/views/pg_reports/dashboard/_show_styles.html.erb
|
|
@@ -166,12 +167,14 @@ files:
|
|
|
166
167
|
- config/locales/uk.yml
|
|
167
168
|
- config/routes.rb
|
|
168
169
|
- lib/pg_reports.rb
|
|
170
|
+
- lib/pg_reports/access_policy.rb
|
|
169
171
|
- lib/pg_reports/annotation_parser.rb
|
|
170
172
|
- lib/pg_reports/compatibility.rb
|
|
171
173
|
- lib/pg_reports/configuration.rb
|
|
172
174
|
- lib/pg_reports/connection/error_translator.rb
|
|
173
175
|
- lib/pg_reports/connection/registry.rb
|
|
174
176
|
- lib/pg_reports/connection/target.rb
|
|
177
|
+
- lib/pg_reports/csv_cell.rb
|
|
175
178
|
- lib/pg_reports/dashboard/reports_registry.rb
|
|
176
179
|
- lib/pg_reports/definitions/connections/active_connections.yml
|
|
177
180
|
- lib/pg_reports/definitions/connections/blocking_queries.yml
|
|
@@ -195,6 +198,7 @@ files:
|
|
|
195
198
|
- lib/pg_reports/definitions/indexes/unused_indexes.yml
|
|
196
199
|
- lib/pg_reports/definitions/queries/all_queries.yml
|
|
197
200
|
- lib/pg_reports/definitions/queries/expensive_queries.yml
|
|
201
|
+
- lib/pg_reports/definitions/queries/frequent_queries.yml
|
|
198
202
|
- lib/pg_reports/definitions/queries/heavy_queries.yml
|
|
199
203
|
- lib/pg_reports/definitions/queries/low_cache_hit_queries.yml
|
|
200
204
|
- lib/pg_reports/definitions/queries/missing_index_queries.yml
|
|
@@ -218,6 +222,7 @@ files:
|
|
|
218
222
|
- lib/pg_reports/definitions/tables/unused_tables.yml
|
|
219
223
|
- lib/pg_reports/definitions/tables/update_hotspots.yml
|
|
220
224
|
- lib/pg_reports/definitions/tables/vacuum_needed.yml
|
|
225
|
+
- lib/pg_reports/drop_index_migration.rb
|
|
221
226
|
- lib/pg_reports/engine.rb
|
|
222
227
|
- lib/pg_reports/error.rb
|
|
223
228
|
- lib/pg_reports/executor.rb
|
|
@@ -233,6 +238,7 @@ files:
|
|
|
233
238
|
- lib/pg_reports/modules/system.rb
|
|
234
239
|
- lib/pg_reports/modules/tables.rb
|
|
235
240
|
- lib/pg_reports/query_monitor.rb
|
|
241
|
+
- lib/pg_reports/rate_limiter.rb
|
|
236
242
|
- lib/pg_reports/report.rb
|
|
237
243
|
- lib/pg_reports/report_definition.rb
|
|
238
244
|
- lib/pg_reports/report_loader.rb
|
|
@@ -256,8 +262,10 @@ files:
|
|
|
256
262
|
- lib/pg_reports/sql/indexes/invalid_indexes.sql
|
|
257
263
|
- lib/pg_reports/sql/indexes/missing_indexes.sql
|
|
258
264
|
- lib/pg_reports/sql/indexes/unused_indexes.sql
|
|
265
|
+
- lib/pg_reports/sql/queries/activity_sample.sql
|
|
259
266
|
- lib/pg_reports/sql/queries/all_queries.sql
|
|
260
267
|
- lib/pg_reports/sql/queries/expensive_queries.sql
|
|
268
|
+
- lib/pg_reports/sql/queries/frequent_queries.sql
|
|
261
269
|
- lib/pg_reports/sql/queries/heavy_queries.sql
|
|
262
270
|
- lib/pg_reports/sql/queries/low_cache_hit_queries.sql
|
|
263
271
|
- lib/pg_reports/sql/queries/missing_index_queries.sql
|
|
@@ -287,6 +295,7 @@ files:
|
|
|
287
295
|
- lib/pg_reports/sql/tables/update_hotspots.sql
|
|
288
296
|
- lib/pg_reports/sql/tables/vacuum_needed.sql
|
|
289
297
|
- lib/pg_reports/sql_loader.rb
|
|
298
|
+
- lib/pg_reports/sql_redactor.rb
|
|
290
299
|
- lib/pg_reports/standalone.rb
|
|
291
300
|
- lib/pg_reports/telegram_sender.rb
|
|
292
301
|
- lib/pg_reports/version.rb
|