pg_canary 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +114 -0
- data/demo.png +0 -0
- data/lib/pg_canary/configuration.rb +80 -0
- data/lib/pg_canary/detector.rb +94 -0
- data/lib/pg_canary/middleware.rb +115 -0
- data/lib/pg_canary/pg_query_support.rb +115 -0
- data/lib/pg_canary/rules/base.rb +77 -0
- data/lib/pg_canary/rules/definitions/array_search_without_gin.rb +75 -0
- data/lib/pg_canary/rules/definitions/cartesian_join.rb +108 -0
- data/lib/pg_canary/rules/definitions/correlated_subquery_in_select.rb +80 -0
- data/lib/pg_canary/rules/definitions/count_star_without_where.rb +57 -0
- data/lib/pg_canary/rules/definitions/deep_offset.rb +63 -0
- data/lib/pg_canary/rules/definitions/distinct_with_join.rb +43 -0
- data/lib/pg_canary/rules/definitions/function_on_column.rb +89 -0
- data/lib/pg_canary/rules/definitions/huge_in_list.rb +74 -0
- data/lib/pg_canary/rules/definitions/implicit_cast.rb +79 -0
- data/lib/pg_canary/rules/definitions/jsonb_search_without_gin.rb +108 -0
- data/lib/pg_canary/rules/definitions/leading_wildcard_like.rb +81 -0
- data/lib/pg_canary/rules/definitions/not_in_subquery.rb +69 -0
- data/lib/pg_canary/rules/definitions/or_across_columns.rb +69 -0
- data/lib/pg_canary/rules/definitions/order_by_random.rb +41 -0
- data/lib/pg_canary/rules/definitions/query_complexity.rb +65 -0
- data/lib/pg_canary/rules/definitions/regex_without_trgm.rb +70 -0
- data/lib/pg_canary/rules/definitions/select_star_with_heavy_columns.rb +73 -0
- data/lib/pg_canary/rules/definitions/unindexed_join.rb +85 -0
- data/lib/pg_canary/rules/definitions/unindexed_order_by_with_limit.rb +51 -0
- data/lib/pg_canary/rules/definitions/unindexed_where.rb +95 -0
- data/lib/pg_canary/rules/definitions/union_instead_of_union_all.rb +30 -0
- data/lib/pg_canary/rules/detection.rb +21 -0
- data/lib/pg_canary/rules/index_predicates.rb +39 -0
- data/lib/pg_canary/rules/query_context.rb +147 -0
- data/lib/pg_canary/schema_introspection.rb +72 -0
- data/lib/pg_canary/subscriber.rb +41 -0
- data/lib/pg_canary/version.rb +5 -0
- data/lib/pg_canary.rb +77 -0
- metadata +126 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 5d97dd5cbefd1e0f736b3800993118237cb01c427e55d7c9dbc44a4296d63627
|
|
4
|
+
data.tar.gz: 1ebaec08d05be5f91fd6275b0236a4b5b6fbd53d826bfd319789cf8ee727f22d
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 539edfe1fe783b0d4619d331e527447054ffd44affc964c514fb6d09005c2f8ffc8b48478816ad1ca088a8ef53b0827165b4895892d50d2222b7f207e46fe29e
|
|
7
|
+
data.tar.gz: 3bae6b432546feb624e0f898f4d8a23e1a3050a25f47e20bbfff89a81e34bd3c4b37acc65fbeb9574fa35e40fd48aab6f23f9d4fa7d99e9593079054149720e6
|
data/CHANGELOG.md
ADDED
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 kyuuri1791
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# pg_canary
|
|
2
|
+
|
|
3
|
+
pg_canary watches the SQL your Rails app executes in development/test and warns about anti-patterns that can become slow queries in production: leading-wildcard `LIKE`s without a trigram index, function-wrapped columns in `WHERE`, `ORDER BY RANDOM()`, `NOT IN (SELECT ...)`, and more. PostgreSQL only.
|
|
4
|
+
|
|
5
|
+

|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
```ruby
|
|
10
|
+
# Gemfile
|
|
11
|
+
group :development, :test do
|
|
12
|
+
gem "pg_canary"
|
|
13
|
+
end
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Detection rules
|
|
17
|
+
|
|
18
|
+
### Tier 1 — enabled by default (suspicious regardless of table size)
|
|
19
|
+
|
|
20
|
+
| Rule | Detects | Rationale |
|
|
21
|
+
|---|---|---|
|
|
22
|
+
| `leading_wildcard_like` | `LIKE '%foo'` / `ILIKE '%foo%'` on a column without a pg_trgm GIN/GiST index | A leading wildcard can never use a btree index (having one doesn't help). pg_trgm is the only fix |
|
|
23
|
+
| `deep_offset` | `OFFSET` beyond a threshold (default 1000) | Every skipped row is read and discarded; deep pages degrade linearly. Suggests keyset pagination |
|
|
24
|
+
| `huge_in_list` | `IN (...)` / `= ANY(...)` with more values than a threshold (default 500) | Expensive to parse/plan, and usually a missing JOIN |
|
|
25
|
+
| `correlated_subquery_in_select` | Scalar subquery in the SELECT list referencing the outer table | Executes once per returned row (an N+1 inside a single query). Suggests JOIN + GROUP BY |
|
|
26
|
+
| `jsonb_search_without_gin` | jsonb search (`@>`, `?`, …) without a GIN index, or `->>` filtering without a matching expression index | No index can serve these predicates otherwise |
|
|
27
|
+
| `array_search_without_gin` | Array search (`@>`, `&&`, `= ANY(column)`) without a GIN index | Same as above, for array columns |
|
|
28
|
+
| `function_on_column` | The searched column wrapped in a function (`WHERE lower(email) = ?`) with no matching expression index | A function-wrapped column cannot be an index search key |
|
|
29
|
+
| `not_in_subquery` | `NOT IN (SELECT ...)` | A single NULL from the subquery silently empties the result, and the planner optimizes it poorly compared to an anti-join. Suggests rewriting to `NOT EXISTS` |
|
|
30
|
+
| `order_by_random` | `ORDER BY RANDOM()` | Reads and sorts every row; cost grows linearly with table size |
|
|
31
|
+
| `regex_without_trgm` | `~` / `~*` / `SIMILAR TO` on a column without a trigram index | The regex variant of `leading_wildcard_like` |
|
|
32
|
+
| `cartesian_join` | JOIN with no join condition — explicit `CROSS JOIN` between tables, or a comma join never connected in WHERE | The result grows with the product of both tables' row counts |
|
|
33
|
+
| `implicit_cast` | Integer column compared with a numeric literal (`age = 1.5`) | The *column* gets implicitly cast to numeric, disabling its index. Restricted to cases provable from the AST alone |
|
|
34
|
+
|
|
35
|
+
Notes:
|
|
36
|
+
|
|
37
|
+
- `deep_offset` and `huge_in_list` read the **runtime bind values** (`OFFSET $1`, array binds) from the `sql.active_record` event — static SQL linters that only see query text cannot do this. `leading_wildcard_like` uses the same mechanism for `LIKE $1`.
|
|
38
|
+
- `function_on_column` matches expression indexes by (function name, column name) — a deliberate approximation rather than full location-insensitive AST equality, which is sufficient in practice
|
|
39
|
+
|
|
40
|
+
### Tier 2 — disabled by default (opt-in)
|
|
41
|
+
|
|
42
|
+
| Rule | Detects |
|
|
43
|
+
|---|---|
|
|
44
|
+
| `unindexed_where` | Equality/range predicate columns in WHERE with no index led by any of them (leftmost-prefix matching on composite indexes is taken into account) |
|
|
45
|
+
| `unindexed_order_by_with_limit` | `ORDER BY x LIMIT n` with no index led by x |
|
|
46
|
+
| `unindexed_join` | Join-condition columns with no index. Looks at joins that actually ran, so raw-SQL joins are covered too |
|
|
47
|
+
| `count_star_without_where` | `SELECT COUNT(*)` without WHERE — MVCC means a full scan. Suggests `reltuples` estimates or counter caches |
|
|
48
|
+
| `distinct_with_join` | DISTINCT combined with JOIN, which often hides join fanout. Suggests `EXISTS`. Legitimate uses exist, hence opt-in |
|
|
49
|
+
| `union_instead_of_union_all` | `UNION` without ALL — deduplication sorts the whole result. Whether duplicates matter is the author's call |
|
|
50
|
+
| `or_across_columns` | OR spanning different columns, which often prevents a single index scan. PostgreSQL can sometimes BitmapOr, hence opt-in |
|
|
51
|
+
| `select_star_with_heavy_columns` | `SELECT *` (ActiveRecord's default) on tables with heavy columns (bytea/text/jsonb, configurable) |
|
|
52
|
+
| `query_complexity` | More than 8 joins or subquery nesting deeper than 4 (both configurable) — the "spaghetti query" guard |
|
|
53
|
+
|
|
54
|
+
**Why disabled by default:** the first four depend on production table size — a missing index is not a problem in itself. Lookup tables like prefectures or plans hold a few dozen rows in production too, and a Seq Scan is the *correct* plan for them. The rest depend on the author's intent (deliberate DISTINCT, required deduplication, acceptable payloads). Nothing in the schema or the AST reveals either, so enabling these rules by default would drown you in false positives.
|
|
55
|
+
|
|
56
|
+
Rule-specific thresholds are set through the same `config.rules` interface:
|
|
57
|
+
|
|
58
|
+
```ruby
|
|
59
|
+
config.rules.deep_offset.threshold = 2000
|
|
60
|
+
config.rules.huge_in_list.threshold = 200
|
|
61
|
+
config.rules.query_complexity.max_joins = 12
|
|
62
|
+
config.rules.select_star_with_heavy_columns.heavy_types = %w[bytea jsonb]
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Once opted in, you can keep false positives down two ways:
|
|
66
|
+
|
|
67
|
+
1. List small tables in `config.ignore_tables`
|
|
68
|
+
2. Provide approximate production row counts in `config.table_size_hints` — when hints are present, Tier 2 rules apply **only** to tables hinted at `size_rule_threshold` (default 10,000) or above. When hints are empty, Tier 2 rules apply to every table — which is precisely why they are off by default.
|
|
69
|
+
|
|
70
|
+
## Configuration
|
|
71
|
+
|
|
72
|
+
Everything works with the defaults; an initializer is only needed to change them — e.g. to enable Tier 2 rules:
|
|
73
|
+
|
|
74
|
+
```ruby
|
|
75
|
+
# config/initializers/pg_canary.rb
|
|
76
|
+
PgCanary.configure do |config|
|
|
77
|
+
config.enabled = Rails.env.development?
|
|
78
|
+
|
|
79
|
+
# Per-rule settings
|
|
80
|
+
config.rules.unindexed_where.enabled = true
|
|
81
|
+
config.rules.leading_wildcard_like.severity = :error # :error / :warning
|
|
82
|
+
|
|
83
|
+
# Excluding tables
|
|
84
|
+
config.ignore_tables = %w[prefectures plans schema_migrations ar_internal_metadata]
|
|
85
|
+
|
|
86
|
+
# For size-dependent (Tier 2) rules: approximate production row counts
|
|
87
|
+
config.table_size_hints = { "users" => 1_000_000, "orders" => 10_000_000 }
|
|
88
|
+
config.size_rule_threshold = 10_000 # apply Tier 2 rules only to tables hinted at or above this
|
|
89
|
+
end
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Notifications
|
|
93
|
+
|
|
94
|
+
Detections show up in a panel at the bottom of the page, each with: the rule name, the SQL, the table/columns involved, why it's a problem, the suggested fix, and the source location (file:line).
|
|
95
|
+
|
|
96
|
+
## How it works
|
|
97
|
+
|
|
98
|
+
pg_canary subscribes to ActiveRecord's `sql.active_record` notifications and analyzes each executed SELECT. Detection is based on the query's AST — parsed with [`pg_query`](https://github.com/pganalyze/pg_query), a binding to PostgreSQL's own parser — combined with schema metadata (index definitions, opclasses, expression indexes, column types) read through ActiveRecord's schema cache. Detections raised during a request are collected by a Rack middleware, which injects the panel into the HTML response.
|
|
99
|
+
|
|
100
|
+
## Development
|
|
101
|
+
|
|
102
|
+
The test suite runs integration tests against a local PostgreSQL (docker-compose included):
|
|
103
|
+
|
|
104
|
+
```console
|
|
105
|
+
$ docker compose up -d --wait
|
|
106
|
+
$ bundle install
|
|
107
|
+
$ bundle exec rake
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
To test against a different Rails version: `RAILS_VERSION=7.1 bundle install && bundle exec rspec`
|
|
111
|
+
|
|
112
|
+
## License
|
|
113
|
+
|
|
114
|
+
The gem is available as open source under the terms of the [MIT License](LICENSE.txt).
|
data/demo.png
ADDED
|
Binary file
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PgCanary
|
|
4
|
+
# Global settings, exposed through PgCanary.configure.
|
|
5
|
+
class Configuration
|
|
6
|
+
# nil means "not explicitly configured" — the Railtie then enables
|
|
7
|
+
# pg_canary in development/test only.
|
|
8
|
+
attr_accessor :enabled
|
|
9
|
+
|
|
10
|
+
# False-positive controls
|
|
11
|
+
attr_accessor :ignore_tables, :table_size_hints, :size_rule_threshold
|
|
12
|
+
|
|
13
|
+
attr_accessor :logger, :app_root
|
|
14
|
+
attr_reader :rules
|
|
15
|
+
|
|
16
|
+
def initialize
|
|
17
|
+
@enabled = nil
|
|
18
|
+
@ignore_tables = %w[schema_migrations ar_internal_metadata]
|
|
19
|
+
@table_size_hints = {}
|
|
20
|
+
@size_rule_threshold = 10_000
|
|
21
|
+
@rules = RulesConfig.new
|
|
22
|
+
@logger = nil
|
|
23
|
+
@app_root = nil
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def ignore_table?(table)
|
|
27
|
+
table && ignore_tables.map(&:to_s).include?(table.to_s)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Whether a size-dependent (Tier 2) rule should look at this table.
|
|
31
|
+
# With no hints configured, every table qualifies (which is exactly why
|
|
32
|
+
# Tier 2 rules are opt-in).
|
|
33
|
+
def size_hint_allows?(table)
|
|
34
|
+
hints = table_size_hints || {}
|
|
35
|
+
return true if hints.empty?
|
|
36
|
+
|
|
37
|
+
hints[table.to_s].to_i >= size_rule_threshold
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Exposes one RuleConfig per built-in rule, both as methods and via [].
|
|
42
|
+
#
|
|
43
|
+
# config.rules.unindexed_where.enabled = true
|
|
44
|
+
# config.rules[:leading_wildcard_like].severity = :error
|
|
45
|
+
class RulesConfig
|
|
46
|
+
def initialize
|
|
47
|
+
@configs = {}
|
|
48
|
+
Rules::Base.all.each do |klass|
|
|
49
|
+
rule_config = RuleConfig.new(klass.options)
|
|
50
|
+
@configs[klass.rule_name] = rule_config
|
|
51
|
+
define_singleton_method(klass.rule_name) { rule_config }
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def [](name)
|
|
56
|
+
@configs.fetch(name.to_sym)
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Per-rule settings. `enabled` / `severity` default to nil, meaning
|
|
61
|
+
# "use the rule class's default". Accessors for rule-specific options
|
|
62
|
+
# (e.g. config.rules.deep_offset.threshold = 2000) are generated from the
|
|
63
|
+
# rule class's declared options, so a typo raises NoMethodError instead of
|
|
64
|
+
# silently creating a setting nobody reads.
|
|
65
|
+
class RuleConfig
|
|
66
|
+
attr_accessor :enabled, :severity
|
|
67
|
+
|
|
68
|
+
def initialize(option_defaults = {})
|
|
69
|
+
@options = option_defaults.dup
|
|
70
|
+
@options.each_key do |name|
|
|
71
|
+
define_singleton_method(name) { @options[name] }
|
|
72
|
+
define_singleton_method(:"#{name}=") { |value| @options[name] = value }
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def [](key)
|
|
77
|
+
@options.fetch(key.to_sym)
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support/backtrace_cleaner"
|
|
4
|
+
|
|
5
|
+
module PgCanary
|
|
6
|
+
# Parses an event's SQL into a QueryContext, runs every enabled rule
|
|
7
|
+
# against it, and filters ignored tables. Each returned Detection carries
|
|
8
|
+
# the query's fingerprint, so a consumer that sees the same query and rule
|
|
9
|
+
# repeatedly (e.g. an N+1 loop within one request) can collapse them —
|
|
10
|
+
# see Middleware.
|
|
11
|
+
class Detector
|
|
12
|
+
include PgQuerySupport
|
|
13
|
+
|
|
14
|
+
# Swallows the configuration whole at construction: pg_canary treats it
|
|
15
|
+
# as fixed after boot (set in an initializer).
|
|
16
|
+
def initialize(config)
|
|
17
|
+
@config = config
|
|
18
|
+
@rules = Rules::Base.all.map(&:new).select { |rule| rule.enabled?(config) }
|
|
19
|
+
@backtrace_cleaner = build_backtrace_cleaner(config.app_root)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# => [Detection] the detections that should be notified.
|
|
23
|
+
def call(payload)
|
|
24
|
+
query = build_query(payload)
|
|
25
|
+
return [] unless query
|
|
26
|
+
|
|
27
|
+
location = nil
|
|
28
|
+
@rules.flat_map do |rule|
|
|
29
|
+
detections = check_rule(rule, query)
|
|
30
|
+
next [] if detections.empty?
|
|
31
|
+
|
|
32
|
+
location ||= source_location
|
|
33
|
+
detections.each do |d|
|
|
34
|
+
d.location = location
|
|
35
|
+
d.fingerprint = query.fingerprint
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
# nil for anything that is not a plain SELECT (or fails to parse).
|
|
43
|
+
def build_query(payload)
|
|
44
|
+
parse_result = PgQuery.parse(payload[:sql])
|
|
45
|
+
stmt = unwrap_node(parse_result.tree.stmts.first&.stmt)
|
|
46
|
+
return nil unless stmt.is_a?(PgQuery::SelectStmt)
|
|
47
|
+
|
|
48
|
+
Rules::QueryContext.new(
|
|
49
|
+
sql: payload[:sql],
|
|
50
|
+
connection: payload[:connection],
|
|
51
|
+
parse_result: parse_result,
|
|
52
|
+
config: @config,
|
|
53
|
+
binds: payload[:binds],
|
|
54
|
+
type_casted_binds: payload[:type_casted_binds]
|
|
55
|
+
)
|
|
56
|
+
rescue PgQuery::ParseError
|
|
57
|
+
nil
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def check_rule(rule, query)
|
|
61
|
+
rule.check(query).reject { |d| ignored?(d, query) }
|
|
62
|
+
rescue StandardError => e
|
|
63
|
+
PgCanary.internal_error(e)
|
|
64
|
+
[]
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def ignored?(detection, query)
|
|
68
|
+
config = query.config
|
|
69
|
+
return true if config.ignore_table?(detection.table)
|
|
70
|
+
|
|
71
|
+
# Table-less detections (e.g. ORDER BY RANDOM() over a subquery) are
|
|
72
|
+
# dropped when every table in the query is ignored.
|
|
73
|
+
if detection.table.nil?
|
|
74
|
+
tables = query.tables
|
|
75
|
+
return tables.any? && tables.all? { |t| config.ignore_table?(t) }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
false
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def build_backtrace_cleaner(root)
|
|
82
|
+
ActiveSupport::BacktraceCleaner.new.tap do |cleaner|
|
|
83
|
+
cleaner.add_silencer { |line| line.include?("lib/pg_canary") }
|
|
84
|
+
cleaner.add_filter { |line| line.delete_prefix("#{root}/") } if root
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# The application frame that triggered the query.
|
|
89
|
+
def source_location
|
|
90
|
+
line = @backtrace_cleaner.clean(caller).first
|
|
91
|
+
line && line[/\A.+?:\d+/]
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "cgi"
|
|
4
|
+
|
|
5
|
+
module PgCanary
|
|
6
|
+
# Rack middleware: collects the detections raised while the request runs
|
|
7
|
+
# (via the thread-local collector .collect appends to), collapses repeats
|
|
8
|
+
# of the same query within that request, and injects a footer panel into
|
|
9
|
+
# HTML responses.
|
|
10
|
+
class Middleware
|
|
11
|
+
COLLECTOR_KEY = :pg_canary_request_detections
|
|
12
|
+
|
|
13
|
+
def self.collect(detections)
|
|
14
|
+
Thread.current[COLLECTOR_KEY]&.concat(detections)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def initialize(app)
|
|
18
|
+
@app = app
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def call(env)
|
|
22
|
+
return @app.call(env) unless active?
|
|
23
|
+
|
|
24
|
+
detections = []
|
|
25
|
+
previous = Thread.current[COLLECTOR_KEY]
|
|
26
|
+
Thread.current[COLLECTOR_KEY] = detections
|
|
27
|
+
status, headers, body = @app.call(env)
|
|
28
|
+
detections = dedup(detections)
|
|
29
|
+
return [status, headers, body] if detections.empty?
|
|
30
|
+
|
|
31
|
+
inject(status, headers, body, detections)
|
|
32
|
+
ensure
|
|
33
|
+
Thread.current[COLLECTOR_KEY] = previous
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def active?
|
|
39
|
+
PgCanary.config.enabled
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Collapses repeats of the same (query, rule) — e.g. an N+1 loop — into
|
|
43
|
+
# a single panel entry. Scoped to this one request: a fresh page load
|
|
44
|
+
# (or a reload of the same page) always shows detections again.
|
|
45
|
+
def dedup(detections)
|
|
46
|
+
detections.uniq { |d| [d.fingerprint, d.rule_name] }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def inject(status, headers, body, detections)
|
|
50
|
+
return [status, headers, body] unless html_response?(headers)
|
|
51
|
+
|
|
52
|
+
content = +""
|
|
53
|
+
body.each { |part| content << part }
|
|
54
|
+
body.close if body.respond_to?(:close)
|
|
55
|
+
|
|
56
|
+
snippet = footer_panel(detections)
|
|
57
|
+
if content.include?("</body>")
|
|
58
|
+
content = content.sub("</body>", "#{snippet}</body>")
|
|
59
|
+
else
|
|
60
|
+
content << snippet
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
headers = headers.merge("Content-Length" => content.bytesize.to_s) if content_length?(headers)
|
|
64
|
+
[status, headers, [content]]
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def html_response?(headers)
|
|
68
|
+
headers.any? { |k, v| k.to_s.casecmp("content-type").zero? && v.to_s.include?("text/html") }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def content_length?(headers)
|
|
72
|
+
headers.any? { |k, _| k.to_s.casecmp("content-length").zero? }
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def footer_panel(detections)
|
|
76
|
+
items = detections.map do |d|
|
|
77
|
+
<<~HTML
|
|
78
|
+
<div style="padding:8px 12px;border-top:1px solid #444;">
|
|
79
|
+
<div>
|
|
80
|
+
<strong style="color:#{d.severity == :error ? "#ff6b6b" : "#ffd93d"};">#{h(d.rule_name)}</strong>
|
|
81
|
+
<span style="opacity:.7;">(#{h(d.severity)})</span>
|
|
82
|
+
#{subject_span(d)}
|
|
83
|
+
</div>
|
|
84
|
+
<pre style="margin:4px 0;white-space:pre-wrap;color:#9ecbff;">#{h(d.truncated_sql)}</pre>
|
|
85
|
+
<div style="white-space:pre-wrap;">→ #{h(d.message)}</div>
|
|
86
|
+
#{%(<pre style="margin:4px 0;white-space:pre-wrap;color:#98c379;">#{h(d.suggestion)}</pre>) if d.suggestion}
|
|
87
|
+
#{%(<div style="opacity:.7;">at #{h(d.location)}</div>) if d.location}
|
|
88
|
+
</div>
|
|
89
|
+
HTML
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
<<~HTML
|
|
93
|
+
<div id="pg-canary-panel" style="position:fixed;left:0;right:0;bottom:0;max-height:45vh;overflow-y:auto;z-index:2147483647;background:#1e1e1e;color:#eee;font:12px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace;box-shadow:0 -2px 8px rgba(0,0,0,.4);">
|
|
94
|
+
<div style="display:flex;justify-content:space-between;align-items:center;padding:6px 12px;background:#2d2d2d;">
|
|
95
|
+
<strong>🐤 PgCanary — #{detections.size} anti-pattern#{"s" if detections.size > 1} detected</strong>
|
|
96
|
+
<button onclick="document.getElementById('pg-canary-panel').remove()" style="background:none;border:1px solid #666;color:#eee;cursor:pointer;padding:2px 8px;">×</button>
|
|
97
|
+
</div>
|
|
98
|
+
#{items.join}
|
|
99
|
+
</div>
|
|
100
|
+
HTML
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def subject_span(detection)
|
|
104
|
+
return "" unless detection.table
|
|
105
|
+
|
|
106
|
+
subject = detection.table
|
|
107
|
+
subject += ".#{detection.columns.join(", ")}" if detection.columns.any?
|
|
108
|
+
%(<span style="opacity:.8;"> — #{h(subject)}</span>)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def h(value)
|
|
112
|
+
CGI.escapeHTML(value.to_s)
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PgCanary
|
|
4
|
+
# Helpers over the pg_query protobuf AST — the unwrapping, traversal and
|
|
5
|
+
# value extraction the pg_query gem does not provide itself. Mix into
|
|
6
|
+
# anything that inspects parsed queries (include for instance level,
|
|
7
|
+
# extend for class level).
|
|
8
|
+
module PgQuerySupport
|
|
9
|
+
COMPARISON_OPS = %w[= <> != < > <= >=].freeze
|
|
10
|
+
|
|
11
|
+
private
|
|
12
|
+
|
|
13
|
+
# --- unwrapping wrapper nodes -------------------------------------
|
|
14
|
+
|
|
15
|
+
# Unwraps a PgQuery::Node (oneof wrapper) into its inner message.
|
|
16
|
+
def unwrap_node(node)
|
|
17
|
+
return nil if node.nil?
|
|
18
|
+
return node unless node.is_a?(PgQuery::Node)
|
|
19
|
+
|
|
20
|
+
node.node ? node.inner : nil
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Strips TypeCast wrappers: `'foo'::text` → the A_Const for 'foo'.
|
|
24
|
+
def strip_type_casts(node)
|
|
25
|
+
node = unwrap_node(node)
|
|
26
|
+
node = unwrap_node(node.arg) while node.is_a?(PgQuery::TypeCast)
|
|
27
|
+
node
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# --- traversal ----------------------------------------------------
|
|
31
|
+
|
|
32
|
+
# Depth-first walk over every protobuf message under +node+, yielding
|
|
33
|
+
# each unwrapped message. When +prune+ returns true for a message, it is
|
|
34
|
+
# yielded but its children are not visited.
|
|
35
|
+
def walk_ast(node, prune: nil, &block)
|
|
36
|
+
node = unwrap_node(node)
|
|
37
|
+
return if node.nil?
|
|
38
|
+
return unless node.is_a?(Google::Protobuf::MessageExts)
|
|
39
|
+
|
|
40
|
+
yield node
|
|
41
|
+
return if prune&.call(node)
|
|
42
|
+
|
|
43
|
+
each_ast_child(node) { |child| walk_ast(child, prune: prune, &block) }
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Yields every message-typed child of a protobuf message.
|
|
47
|
+
def each_ast_child(node, &block)
|
|
48
|
+
node.class.descriptor.each do |field|
|
|
49
|
+
next unless field.type == :message
|
|
50
|
+
|
|
51
|
+
value = node[field.name]
|
|
52
|
+
next if value.nil?
|
|
53
|
+
|
|
54
|
+
if value.is_a?(Google::Protobuf::RepeatedField)
|
|
55
|
+
value.each(&block)
|
|
56
|
+
else
|
|
57
|
+
yield value
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Walk that does not descend into nested sub-selects, so each scope is
|
|
63
|
+
# analyzed exactly once. Yielding starts at the given node's children,
|
|
64
|
+
# i.e. pass a where_clause / sort item, not a whole statement.
|
|
65
|
+
def walk_within_scope(node, &)
|
|
66
|
+
root_seen = false
|
|
67
|
+
walk_ast(node, prune: lambda { |msg|
|
|
68
|
+
if root_seen
|
|
69
|
+
msg.is_a?(PgQuery::SelectStmt)
|
|
70
|
+
else
|
|
71
|
+
root_seen = true # never prune the root itself
|
|
72
|
+
false
|
|
73
|
+
end
|
|
74
|
+
}, &)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# --- reading values out of nodes ----------------------------------
|
|
78
|
+
|
|
79
|
+
# ["users", "name"] for "users"."name"; nil when the ref contains A_Star.
|
|
80
|
+
def column_ref_fields(column_ref)
|
|
81
|
+
fields = column_ref.fields.map { |f| unwrap_node(f) }
|
|
82
|
+
return nil unless fields.all?(PgQuery::String)
|
|
83
|
+
|
|
84
|
+
fields.map(&:sval)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def string_values(nodes)
|
|
88
|
+
nodes.map { |n| unwrap_node(n) }.grep(PgQuery::String).map(&:sval)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Operator name of an A_Expr, e.g. "=", "~~", "~~*".
|
|
92
|
+
def operator_name(a_expr)
|
|
93
|
+
string_values(a_expr.name).last
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Unqualified, downcased function name of a FuncCall.
|
|
97
|
+
def function_name(func_call)
|
|
98
|
+
string_values(func_call.funcname).last&.downcase
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Ruby value of an A_Const: Integer, Float, String, true/false or nil.
|
|
102
|
+
def constant_value(a_const)
|
|
103
|
+
case a_const.val
|
|
104
|
+
when :ival then a_const.ival.ival
|
|
105
|
+
when :fval then a_const.fval.fval.to_f
|
|
106
|
+
when :sval then a_const.sval.sval
|
|
107
|
+
when :boolval then a_const.boolval.boolval
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def comparison_expr?(a_expr)
|
|
112
|
+
a_expr.kind == :AEXPR_OP && COMPARISON_OPS.include?(operator_name(a_expr))
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PgCanary
|
|
4
|
+
module Rules
|
|
5
|
+
# Base class for detection rules.
|
|
6
|
+
#
|
|
7
|
+
# A rule implements #check(query) and returns an array of Detection.
|
|
8
|
+
# Configuration flows in explicitly through the QueryContext
|
|
9
|
+
# (query.config); rules never reach for global state themselves.
|
|
10
|
+
class Base
|
|
11
|
+
include PgQuerySupport
|
|
12
|
+
|
|
13
|
+
class << self
|
|
14
|
+
def all
|
|
15
|
+
subclasses.sort_by(&:name)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# :leading_wildcard_like for LeadingWildcardLike
|
|
19
|
+
def rule_name
|
|
20
|
+
@rule_name ||= name.split("::").last
|
|
21
|
+
.gsub(/([a-z\d])([A-Z])/, '\1_\2')
|
|
22
|
+
.downcase.to_sym
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Rule-specific options and their defaults, e.g. { threshold: 1000 }.
|
|
26
|
+
# Declared statically so RuleConfig can generate real accessors.
|
|
27
|
+
def options
|
|
28
|
+
{}
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def check(_query)
|
|
33
|
+
raise NotImplementedError, "#{self.class} must implement #check"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def enabled?(config)
|
|
37
|
+
setting = config.rules[self.class.rule_name].enabled
|
|
38
|
+
setting.nil? ? default_enabled : setting
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def default_enabled
|
|
42
|
+
raise NotImplementedError, "#{self.class} must declare #default_enabled (true for Tier 1, false for Tier 2)"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def size_dependent?
|
|
46
|
+
raise NotImplementedError, "#{self.class} must declare #size_dependent?"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def rule_config(query)
|
|
52
|
+
query.config.rules[self.class.rule_name]
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Tier 2 gate: with size hints configured, only tables hinted as large
|
|
56
|
+
# are checked.
|
|
57
|
+
def applicable_table?(query, table)
|
|
58
|
+
return false if query.config.ignore_table?(table)
|
|
59
|
+
return true unless size_dependent?
|
|
60
|
+
|
|
61
|
+
query.config.size_hint_allows?(table)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def detection(query, message:, suggestion: nil, table: nil, columns: nil)
|
|
65
|
+
Detection.new(
|
|
66
|
+
rule_name: self.class.rule_name,
|
|
67
|
+
severity: rule_config(query).severity || :warning,
|
|
68
|
+
sql: query.sql,
|
|
69
|
+
table: table,
|
|
70
|
+
columns: Array(columns),
|
|
71
|
+
message: message,
|
|
72
|
+
suggestion: suggestion
|
|
73
|
+
)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|