schema_reaper 1.0.16 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +98 -0
- data/PRO.md +11 -6
- data/README.md +368 -71
- data/lib/generators/schema_reaper/install/install_generator.rb +194 -0
- data/lib/generators/schema_reaper/install/templates/initializer.rb.tt +18 -0
- data/lib/generators/schema_reaper/install/templates/schema_reaper_controller.rb.tt +69 -0
- data/lib/schema_reaper/alert_config.rb +55 -0
- data/lib/schema_reaper/analyzers/dead_column.rb +10 -4
- data/lib/schema_reaper/introspect/postgres.rb +22 -4
- data/lib/schema_reaper/mailer.rb +31 -0
- data/lib/schema_reaper/notifier.rb +106 -0
- data/lib/schema_reaper/scan_job.rb +39 -0
- data/lib/schema_reaper/tasks/schema_reaper.rake +5 -0
- data/lib/schema_reaper/version.rb +1 -1
- data/lib/schema_reaper.rb +14 -0
- data/schema_reaper.gemspec +5 -1
- metadata +83 -2
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
|
|
6
|
+
module SchemaReaper
|
|
7
|
+
module Generators
|
|
8
|
+
# `rails generate schema_reaper:install` -- one-time setup for unattended
|
|
9
|
+
# production scanning: an initializer for where reports go, a scheduled
|
|
10
|
+
# job (whenever or sidekiq-cron, whichever the app already uses), and a
|
|
11
|
+
# token-protected manual-trigger route. See the plan this implements:
|
|
12
|
+
# production automation with zero ongoing DevOps dependency -- a normal
|
|
13
|
+
# `git commit` + deploy is the only step after this generator runs.
|
|
14
|
+
class InstallGenerator < Rails::Generators::Base
|
|
15
|
+
source_root File.expand_path("templates", __dir__)
|
|
16
|
+
|
|
17
|
+
def create_initializer
|
|
18
|
+
template "initializer.rb.tt", "config/initializers/schema_reaper.rb"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def create_manual_trigger
|
|
22
|
+
template "schema_reaper_controller.rb.tt", "app/controllers/schema_reaper_controller.rb"
|
|
23
|
+
|
|
24
|
+
# `route` (unlike `template`/`create_file`) has no conflict
|
|
25
|
+
# detection of its own -- it unconditionally injects, so re-running
|
|
26
|
+
# this generator would duplicate the line every time without this
|
|
27
|
+
# guard.
|
|
28
|
+
routes_path = File.join(destination_root, "config/routes.rb")
|
|
29
|
+
if File.exist?(routes_path) && File.read(routes_path).include?("schema_reaper#trigger")
|
|
30
|
+
say_status :skip, "config/routes.rb already routes to schema_reaper#trigger -- not duplicating it", :yellow
|
|
31
|
+
else
|
|
32
|
+
route "post '/internal/schema_scan', to: 'schema_reaper#trigger'"
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def add_schedule
|
|
37
|
+
case detected_scheduler
|
|
38
|
+
when :whenever then add_whenever_schedule
|
|
39
|
+
when :sidekiq_cron then add_sidekiq_cron_schedule
|
|
40
|
+
else
|
|
41
|
+
say_status :skip, "no `whenever` or `sidekiq-cron` gem detected -- add a cron entry yourself " \
|
|
42
|
+
"that runs `rake schema_reaper:alert` on whatever cadence you want", :yellow
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def warn_if_dev_scoped
|
|
47
|
+
return unless gemfile_scopes_schema_reaper_to_dev?
|
|
48
|
+
|
|
49
|
+
say_status :warning,
|
|
50
|
+
"schema_reaper is Gemfile-scoped to group: :development. Most production deploy " \
|
|
51
|
+
"pipelines strip dev/test groups (`BUNDLE_WITHOUT=development:test`), so " \
|
|
52
|
+
"the scheduled scan will not run in production until you remove that restriction.",
|
|
53
|
+
:red
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def warn_if_no_mailer_from
|
|
57
|
+
return if mailer_from_configured?
|
|
58
|
+
|
|
59
|
+
say_status :warning,
|
|
60
|
+
"no ApplicationMailer default `from:` detected (nor config.action_mailer.default_options). " \
|
|
61
|
+
"SchemaReaper::Mailer falls back to a " \
|
|
62
|
+
"placeholder sender in that case, which most real SMTP relays reject or spam-flag -- " \
|
|
63
|
+
"email reports would silently fail to arrive. If you plan to use the email channel " \
|
|
64
|
+
"(config.emails in the initializer this generator just wrote), set " \
|
|
65
|
+
"`default from: \"...\"` on ApplicationMailer first.",
|
|
66
|
+
:red
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def print_token_instructions
|
|
70
|
+
token = "#{app_identifier}-#{SecureRandom.hex(32)}"
|
|
71
|
+
|
|
72
|
+
say ""
|
|
73
|
+
say "Manual trigger token (shown once -- store it now):", :green
|
|
74
|
+
say " #{token}"
|
|
75
|
+
say ""
|
|
76
|
+
say "This is genuinely sensitive (it authorizes a production action), so it does not go in a " \
|
|
77
|
+
"committed file. Store it in encrypted credentials instead:"
|
|
78
|
+
say ""
|
|
79
|
+
say " rails credentials:edit"
|
|
80
|
+
say ""
|
|
81
|
+
say "and add:"
|
|
82
|
+
say ""
|
|
83
|
+
say " schema_reaper:"
|
|
84
|
+
say " trigger_token: #{token}"
|
|
85
|
+
say ""
|
|
86
|
+
say "Trigger a scan on demand with:"
|
|
87
|
+
say ""
|
|
88
|
+
say " curl -X POST https://your-app.example.com/internal/schema_scan \\"
|
|
89
|
+
say " -H \"Authorization: Bearer #{token}\""
|
|
90
|
+
say ""
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
private
|
|
94
|
+
|
|
95
|
+
def detected_scheduler
|
|
96
|
+
return :whenever if gem_locked?("whenever")
|
|
97
|
+
return :sidekiq_cron if gem_locked?("sidekiq-cron")
|
|
98
|
+
|
|
99
|
+
nil
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def gem_locked?(name)
|
|
103
|
+
Bundler.locked_gems&.specs&.any? { |s| s.name == name } || false
|
|
104
|
+
rescue StandardError
|
|
105
|
+
false
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def add_whenever_schedule
|
|
109
|
+
path = "config/schedule.rb"
|
|
110
|
+
marker = 'rake "schema_reaper:alert"'
|
|
111
|
+
entry = <<~RUBY
|
|
112
|
+
|
|
113
|
+
every 3.months do
|
|
114
|
+
#{marker}
|
|
115
|
+
end
|
|
116
|
+
RUBY
|
|
117
|
+
|
|
118
|
+
full_path = File.join(destination_root, path)
|
|
119
|
+
if File.exist?(full_path) && File.read(full_path).include?(marker)
|
|
120
|
+
say_status :skip, "#{path} already has a schema_reaper entry -- not duplicating it", :yellow
|
|
121
|
+
elsif File.exist?(full_path)
|
|
122
|
+
append_to_file path, entry
|
|
123
|
+
else
|
|
124
|
+
# No `require "whenever"` here -- real wheneverize-generated files
|
|
125
|
+
# don't have one; whenever's own CLI evaluates this file through
|
|
126
|
+
# its DSL, not as a plain script that needs to load itself.
|
|
127
|
+
create_file path, entry.sub("\n\n", "")
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def add_sidekiq_cron_schedule
|
|
132
|
+
path = "config/schedule.yml"
|
|
133
|
+
marker = "schema_reaper_scan:"
|
|
134
|
+
entry = <<~YAML
|
|
135
|
+
|
|
136
|
+
#{marker}
|
|
137
|
+
cron: "0 4 1 */3 *" # 4am on the 1st, every 3 months
|
|
138
|
+
class: "SchemaReaper::ScanJob"
|
|
139
|
+
queue: default
|
|
140
|
+
active_job: true # explicit, not relying on class-ancestry auto-detection
|
|
141
|
+
YAML
|
|
142
|
+
|
|
143
|
+
full_path = File.join(destination_root, path)
|
|
144
|
+
if File.exist?(full_path) && File.read(full_path).include?(marker)
|
|
145
|
+
say_status :skip, "#{path} already has a schema_reaper_scan entry -- not duplicating it", :yellow
|
|
146
|
+
elsif File.exist?(full_path)
|
|
147
|
+
append_to_file path, entry
|
|
148
|
+
else
|
|
149
|
+
create_file path, entry.sub("\n\n", "")
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# `gem "schema_reaper", group: :development` (or the equivalent
|
|
154
|
+
# `group :development do ... end` block form) in the app's own Gemfile
|
|
155
|
+
# -- see README's current install snippet. A dev-scoped gem is absent
|
|
156
|
+
# from `BUNDLE_WITHOUT=development:test` installs, which most
|
|
157
|
+
# production deploy pipelines run, so the scheduled scan silently
|
|
158
|
+
# never runs.
|
|
159
|
+
#
|
|
160
|
+
# Parsed with Bundler's own DSL rather than line-scanning for
|
|
161
|
+
# "group: :development" -- a regex over raw lines misses the block
|
|
162
|
+
# form entirely (the gem's own line never mentions :development; the
|
|
163
|
+
# `group :development do` line above it does), and that block form is
|
|
164
|
+
# the more common style in practice, not an edge case.
|
|
165
|
+
def gemfile_scopes_schema_reaper_to_dev?
|
|
166
|
+
gemfile = File.join(destination_root, "Gemfile")
|
|
167
|
+
return false unless File.exist?(gemfile)
|
|
168
|
+
|
|
169
|
+
dep = Bundler::Dsl.evaluate(gemfile, nil, {}).dependencies.find { |d| d.name == "schema_reaper" }
|
|
170
|
+
dep && !dep.groups.include?(:default)
|
|
171
|
+
rescue StandardError
|
|
172
|
+
false # a Gemfile we can't parse shouldn't block the rest of the generator
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Checked against the class SchemaReaper::Mailer will inherit from --
|
|
176
|
+
# ApplicationMailer when the app has one, ActionMailer::Base otherwise
|
|
177
|
+
# (where `config.action_mailer.default_options = { from: ... }` lands).
|
|
178
|
+
# At generate time the app is already booted, so this reads the same
|
|
179
|
+
# `default_params[:from]` chain Mailer itself will see.
|
|
180
|
+
def mailer_from_configured?
|
|
181
|
+
parent = defined?(::ApplicationMailer) ? ::ApplicationMailer : ::ActionMailer::Base
|
|
182
|
+
parent.default_params[:from].present?
|
|
183
|
+
rescue StandardError
|
|
184
|
+
true # can't determine -- don't nag over something we're unsure about
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def app_identifier
|
|
188
|
+
Rails.application.class.module_parent_name.underscore
|
|
189
|
+
rescue StandardError
|
|
190
|
+
"app"
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# schema_reaper production automation.
|
|
4
|
+
#
|
|
5
|
+
# Neither of these is a secret -- leaking a teammate's email address or a
|
|
6
|
+
# webhook URL isn't a meaningful security event -- so this file is meant to
|
|
7
|
+
# be committed to git directly. The manual-trigger token is not: see the
|
|
8
|
+
# instructions `rails generate schema_reaper:install` printed for that one
|
|
9
|
+
# (it lives in Rails.application.credentials, not here).
|
|
10
|
+
#
|
|
11
|
+
# Uncomment and fill in at least one of these to start receiving reports.
|
|
12
|
+
# Both fire on every scheduled run when both are set -- this isn't a
|
|
13
|
+
# fallback, it's two audiences: a dev-facing chat channel and a formal inbox
|
|
14
|
+
# record for people who don't watch chat.
|
|
15
|
+
SchemaReaper::AlertConfig.configure do |config|
|
|
16
|
+
# config.emails = %w[dev1@example.com dev2@example.com]
|
|
17
|
+
# config.webhook_url = "https://hooks.slack.com/services/T000/B000/XXXX"
|
|
18
|
+
end
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Manual, on-demand trigger for the production schema_reaper scan -- enqueues
|
|
4
|
+
# the exact same job the schedule uses, so there is one code path to keep
|
|
5
|
+
# correct. Protected by a bearer token stored in encrypted credentials
|
|
6
|
+
# (config/credentials.yml.enc, key schema_reaper.trigger_token) rather than a
|
|
7
|
+
# DevOps-provisioned ENV var -- every Rails app already has the
|
|
8
|
+
# RAILS_MASTER_KEY needed to decrypt it, so this introduces zero new
|
|
9
|
+
# production configuration.
|
|
10
|
+
#
|
|
11
|
+
# Trigger it with (the token is printed once by the install generator --
|
|
12
|
+
# store it somewhere your team can find it, e.g. a password manager. It is
|
|
13
|
+
# deliberately NOT embedded in the scheduled report itself: that report goes
|
|
14
|
+
# to Slack/email, and a bearer token that authorizes a production action
|
|
15
|
+
# doesn't belong in either):
|
|
16
|
+
#
|
|
17
|
+
# curl -X POST https://your-app.example.com/internal/schema_scan \
|
|
18
|
+
# -H "Authorization: Bearer <token>"
|
|
19
|
+
class SchemaReaperController < ActionController::Base
|
|
20
|
+
# This is a token-authenticated API endpoint, not a session-backed form --
|
|
21
|
+
# Rails' CSRF protection is meant for the latter. Deliberately does NOT
|
|
22
|
+
# inherit from ::ApplicationController, and explicitly skips the
|
|
23
|
+
# authenticity-token check (`raise: false` so this doesn't itself blow up
|
|
24
|
+
# on an app where default_protect_from_forgery is off and the callback was
|
|
25
|
+
# never registered). Without this, a plain curl POST is rejected with
|
|
26
|
+
# ActionController::InvalidAuthenticityToken before the token check below
|
|
27
|
+
# even runs -- Rails wires `protect_from_forgery with: :exception` onto
|
|
28
|
+
# ActionController::Base itself for every app using Rails' post-5.2
|
|
29
|
+
# defaults, so every subclass inherits it, this one included.
|
|
30
|
+
skip_before_action :verify_authenticity_token, raise: false
|
|
31
|
+
|
|
32
|
+
# A valid (or leaked) token could otherwise be POSTed repeatedly, each hit
|
|
33
|
+
# enqueueing a full schema introspection against production. This is not
|
|
34
|
+
# per-caller rate limiting -- it's a single shared cooldown so a burst of
|
|
35
|
+
# triggers (accidental double-click, a leaked token being hammered) can't
|
|
36
|
+
# stack up scans faster than they can plausibly be useful. Rails.cache is
|
|
37
|
+
# whatever the app already has configured; on the default NullStore (no
|
|
38
|
+
# caching configured at all) this is a no-op, same as before this guard
|
|
39
|
+
# existed -- configure a real cache store to get actual protection.
|
|
40
|
+
TRIGGER_COOLDOWN = 5.minutes
|
|
41
|
+
|
|
42
|
+
def trigger
|
|
43
|
+
return head(:unauthorized) unless token_valid?
|
|
44
|
+
return head(:too_many_requests) unless acquire_trigger_lock
|
|
45
|
+
|
|
46
|
+
SchemaReaper::ScanJob.perform_later
|
|
47
|
+
head :accepted
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def acquire_trigger_lock
|
|
53
|
+
Rails.cache.write("schema_reaper:manual_trigger_lock", true, unless_exist: true, expires_in: TRIGGER_COOLDOWN)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def token_valid?
|
|
57
|
+
expected = Rails.application.credentials.dig(:schema_reaper, :trigger_token)
|
|
58
|
+
return false if expected.blank?
|
|
59
|
+
|
|
60
|
+
provided = request.headers["Authorization"].to_s.delete_prefix("Bearer ").strip
|
|
61
|
+
return false if provided.empty?
|
|
62
|
+
|
|
63
|
+
ActiveSupport::SecurityUtils.secure_compare(provided, expected)
|
|
64
|
+
rescue ActiveSupport::EncryptedFile::MissingKeyError, ActiveSupport::MessageEncryptor::InvalidMessage
|
|
65
|
+
# RAILS_MASTER_KEY absent or credentials undecryptable in this
|
|
66
|
+
# environment -- fail closed (unauthorized), don't 500.
|
|
67
|
+
false
|
|
68
|
+
end
|
|
69
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SchemaReaper
|
|
4
|
+
# Where the production scan's report goes. Set once from a Rails
|
|
5
|
+
# initializer (config/initializers/schema_reaper.rb), typically written by
|
|
6
|
+
# `rails generate schema_reaper:install`:
|
|
7
|
+
#
|
|
8
|
+
# SchemaReaper::AlertConfig.configure do |config|
|
|
9
|
+
# config.emails = %w[dev1@example.com dev2@example.com]
|
|
10
|
+
# config.webhook_url = "https://hooks.slack.com/services/T000/B000/XXXX"
|
|
11
|
+
# end
|
|
12
|
+
#
|
|
13
|
+
# Neither field is a secret -- leaking a teammate's email address or a
|
|
14
|
+
# webhook URL isn't a meaningful security event -- so this is meant to be
|
|
15
|
+
# committed to git directly. The manual-trigger token is not: that one
|
|
16
|
+
# lives in Rails.application.credentials (see the install generator).
|
|
17
|
+
class AlertConfig
|
|
18
|
+
class << self
|
|
19
|
+
def configure
|
|
20
|
+
yield instance
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def instance
|
|
24
|
+
@instance ||= new
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Test helper -- config set in one example must not leak into the next.
|
|
28
|
+
def reset!
|
|
29
|
+
@instance = new
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
attr_accessor :emails, :webhook_url
|
|
34
|
+
|
|
35
|
+
def initialize
|
|
36
|
+
@emails = []
|
|
37
|
+
@webhook_url = nil
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def emails?
|
|
41
|
+
!emails.to_a.empty?
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def webhook?
|
|
45
|
+
!webhook_url.to_s.strip.empty?
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Whether Notifier has anywhere to send a report. A generator-scaffolded
|
|
49
|
+
# but unfilled-in config (both fields left commented out) is a safe
|
|
50
|
+
# no-op, not an error.
|
|
51
|
+
def configured?
|
|
52
|
+
emails? || webhook?
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -4,12 +4,12 @@ module SchemaReaper
|
|
|
4
4
|
module Analyzers
|
|
5
5
|
# Flags columns present in the schema but never referenced in code. When a
|
|
6
6
|
# runtime usage log is supplied, its signal is fused in: a column unseen in
|
|
7
|
-
# BOTH code and
|
|
7
|
+
# BOTH code and at least `min_age_days` (default 14) observed days of
|
|
8
|
+
# runtime reaches high confidence.
|
|
8
9
|
class DeadColumn < Base
|
|
9
10
|
Registry.register(self)
|
|
10
11
|
|
|
11
|
-
STATIC_ONLY_CAP
|
|
12
|
-
RUNTIME_MIN_DAYS = 14
|
|
12
|
+
STATIC_ONLY_CAP = 0.6
|
|
13
13
|
|
|
14
14
|
def call
|
|
15
15
|
schema.tables.reject { |t| config.ignore_tables.include?(t.name) }
|
|
@@ -48,7 +48,7 @@ module SchemaReaper
|
|
|
48
48
|
end
|
|
49
49
|
|
|
50
50
|
def confidence_for(col)
|
|
51
|
-
if runtime.present? && runtime.observed_days >=
|
|
51
|
+
if runtime.present? && runtime.observed_days >= min_runtime_days
|
|
52
52
|
col.null ? 0.9 : 0.8
|
|
53
53
|
else
|
|
54
54
|
base = col.null ? 0.5 : 0.4
|
|
@@ -56,6 +56,12 @@ module SchemaReaper
|
|
|
56
56
|
end
|
|
57
57
|
end
|
|
58
58
|
|
|
59
|
+
# How many days of runtime data it takes before "never read at runtime"
|
|
60
|
+
# is trusted. A config built without the key (nil) keeps the default.
|
|
61
|
+
def min_runtime_days
|
|
62
|
+
config.min_age_days || Config::DEFAULTS["min_age_days"]
|
|
63
|
+
end
|
|
64
|
+
|
|
59
65
|
def evidence_for(table, col)
|
|
60
66
|
ev = ["no `#{col.name}` reference found in scanned code"]
|
|
61
67
|
ev << "column is nullable" if col.null
|
|
@@ -1,7 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module SchemaReaper
|
|
4
|
+
# Reads a live database's schema and statistics. PostgreSQL only for now.
|
|
4
5
|
module Introspect
|
|
6
|
+
# `pg` is deliberately not a runtime dependency -- the host app already
|
|
7
|
+
# bundles its own (every Rails + PostgreSQL app does), and pinning a second
|
|
8
|
+
# version here would fight it. Without this rescue a missing gem surfaces
|
|
9
|
+
# as a baffling NameError instead.
|
|
10
|
+
def self.require_pg!
|
|
11
|
+
require "pg"
|
|
12
|
+
rescue LoadError
|
|
13
|
+
raise Error, "schema_reaper needs the `pg` gem to talk to PostgreSQL -- add `gem \"pg\"` to your Gemfile"
|
|
14
|
+
end
|
|
15
|
+
|
|
5
16
|
# Reads live schema + planner statistics from PostgreSQL using the `pg` gem
|
|
6
17
|
# directly, so the host app does not need to boot Rails.
|
|
7
18
|
class Postgres
|
|
@@ -22,12 +33,10 @@ module SchemaReaper
|
|
|
22
33
|
COLUMN_SEPARATOR = 31.chr
|
|
23
34
|
|
|
24
35
|
def initialize(url)
|
|
25
|
-
|
|
36
|
+
Introspect.require_pg!
|
|
26
37
|
raise Error, NO_URL if url.nil? || url.empty?
|
|
27
38
|
|
|
28
|
-
@conn =
|
|
29
|
-
rescue PG::Error => e
|
|
30
|
-
raise Error, "could not connect to the database: #{e.message.strip}"
|
|
39
|
+
@conn = connect(url)
|
|
31
40
|
end
|
|
32
41
|
|
|
33
42
|
def call
|
|
@@ -39,6 +48,15 @@ module SchemaReaper
|
|
|
39
48
|
|
|
40
49
|
private
|
|
41
50
|
|
|
51
|
+
# Kept out of #initialize: a `rescue PG::Error` there is evaluated for
|
|
52
|
+
# *any* exception, including require_pg!'s, and would itself raise NameError
|
|
53
|
+
# when pg is the thing that failed to load.
|
|
54
|
+
def connect(url)
|
|
55
|
+
PG.connect(url)
|
|
56
|
+
rescue PG::Error => e
|
|
57
|
+
raise Error, "could not connect to the database: #{e.message.strip}"
|
|
58
|
+
end
|
|
59
|
+
|
|
42
60
|
# Cluster-wide cumulative index scans. Lets the unused-index analyzer tell
|
|
43
61
|
# "this index is never used" apart from "this database has no query
|
|
44
62
|
# history", which look identical at the level of a single idx_scan = 0.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SchemaReaper
|
|
4
|
+
# Inherits ApplicationMailer when the host app defines one (the normal
|
|
5
|
+
# case), so it picks up the app's own `default from:`, layout and delivery
|
|
6
|
+
# settings automatically -- "reuse the app's already-configured mailer,
|
|
7
|
+
# don't require a new SMTP setup" is the whole point of the email channel.
|
|
8
|
+
# Falls back to ActionMailer::Base directly for apps that don't.
|
|
9
|
+
class Mailer < (defined?(::ApplicationMailer) ? ::ApplicationMailer : ActionMailer::Base)
|
|
10
|
+
# Only applied when nothing upstream (ApplicationMailer or its own
|
|
11
|
+
# ancestors) already set one. Without this, `mail()` raises "SMTP From
|
|
12
|
+
# address may not be blank" -- and Notifier's rescue never sees it:
|
|
13
|
+
# deliver_later only enqueues, the actual render/from-validation happens
|
|
14
|
+
# later inside ActionMailer::MailDeliveryJob#perform, so an app that
|
|
15
|
+
# hits this path gets a bare, unbranded ActiveJob failure with no
|
|
16
|
+
# `[schema_reaper]` log line anywhere near it. This placeholder isn't a
|
|
17
|
+
# real deliverable address -- most SMTP relays will still reject or
|
|
18
|
+
# spam-flag mail from an unconfigured sender -- it only turns a hard
|
|
19
|
+
# crash into a normal delivery failure. The install generator warns at
|
|
20
|
+
# setup time when this fallback would apply, so the real fix (configure
|
|
21
|
+
# a `default from:`) doesn't have to be discovered from a stack trace.
|
|
22
|
+
default from: "schema_reaper@localhost" unless default_params[:from]
|
|
23
|
+
|
|
24
|
+
def report_email(to:, report:)
|
|
25
|
+
@report = report
|
|
26
|
+
mail(to: to, subject: "[schema_reaper] scan report") do |format|
|
|
27
|
+
format.text { render plain: @report }
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
require "json"
|
|
5
|
+
require "stringio"
|
|
6
|
+
require_relative "reporters/markdown"
|
|
7
|
+
|
|
8
|
+
module SchemaReaper
|
|
9
|
+
# Delivers a completed scan's findings through whichever channels are
|
|
10
|
+
# configured in AlertConfig -- a Slack-compatible webhook POST and/or email
|
|
11
|
+
# via the host app's own ActionMailer setup. Both fire when both are
|
|
12
|
+
# configured; this isn't a fallback chain (schema reports aren't critical
|
|
13
|
+
# enough to need one), it's reaching two different audiences: a dev-facing
|
|
14
|
+
# chat channel and a more formal inbox record for people who don't watch
|
|
15
|
+
# chat.
|
|
16
|
+
#
|
|
17
|
+
# Delivery failures are logged, never raised -- a webhook endpoint being
|
|
18
|
+
# briefly down should not fail the scan job or retry-storm it.
|
|
19
|
+
class Notifier
|
|
20
|
+
# `mailer:` defaults to :auto rather than nil so an explicit
|
|
21
|
+
# `mailer: nil` (forcing "no mailer available", e.g. to test that path,
|
|
22
|
+
# or because a caller genuinely wants email delivery skipped) is
|
|
23
|
+
# respected rather than silently falling back to the auto-detected
|
|
24
|
+
# SchemaReaper::Mailer via `||`.
|
|
25
|
+
#
|
|
26
|
+
# `mail_delivery:` is :later (enqueue via ActiveJob, the normal case) or
|
|
27
|
+
# :now -- for a caller that is itself running inline in a short-lived
|
|
28
|
+
# process, where an in-process queue would be torn down before the mail
|
|
29
|
+
# job ever ran (see the schema_reaper:alert rake task).
|
|
30
|
+
def initialize(findings, config: AlertConfig.instance, http: Net::HTTP, mailer: :auto, mail_delivery: :later)
|
|
31
|
+
@findings = findings
|
|
32
|
+
@config = config
|
|
33
|
+
@http = http
|
|
34
|
+
@mailer = mailer == :auto ? default_mailer : mailer
|
|
35
|
+
@mail_delivery = mail_delivery
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def deliver
|
|
39
|
+
return unless @config.configured?
|
|
40
|
+
|
|
41
|
+
deliver_webhook if @config.webhook?
|
|
42
|
+
deliver_email if @config.emails?
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
# Mailer is autoloaded (see lib/schema_reaper.rb), and it can only be
|
|
48
|
+
# defined when ActionMailer is present -- check that, not Mailer itself.
|
|
49
|
+
def default_mailer
|
|
50
|
+
defined?(::ActionMailer::Base) ? Mailer : nil
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def report_text
|
|
54
|
+
@report_text ||= begin
|
|
55
|
+
io = StringIO.new
|
|
56
|
+
Reporters::Markdown.new(@findings, io: io).render
|
|
57
|
+
io.string
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Slack's incoming-webhook shape ({"text": ...}) is the most common
|
|
62
|
+
# receiver in practice; Discord's legacy webhook path also accepts a
|
|
63
|
+
# plain "content" body but that's a different key, and structured-alert
|
|
64
|
+
# services like PagerDuty need an entirely different schema (their
|
|
65
|
+
# Events API v2, not a text webhook) -- those need a different endpoint,
|
|
66
|
+
# not this one.
|
|
67
|
+
def deliver_webhook
|
|
68
|
+
uri = URI.parse(@config.webhook_url)
|
|
69
|
+
request = Net::HTTP::Post.new(uri)
|
|
70
|
+
request["Content-Type"] = "application/json"
|
|
71
|
+
request.body = JSON.generate(text: report_text)
|
|
72
|
+
|
|
73
|
+
client = @http.new(uri.host, uri.port)
|
|
74
|
+
client.use_ssl = uri.scheme == "https"
|
|
75
|
+
# A background job blocking indefinitely on a dead webhook host would
|
|
76
|
+
# back up the queue behind it -- bound the wait instead of relying on
|
|
77
|
+
# Net::HTTP's own (version-dependent) defaults.
|
|
78
|
+
client.open_timeout = 10
|
|
79
|
+
client.read_timeout = 10
|
|
80
|
+
client.request(request)
|
|
81
|
+
rescue StandardError => e
|
|
82
|
+
log_error("webhook delivery failed", e)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def deliver_email
|
|
86
|
+
unless @mailer
|
|
87
|
+
log_error("email delivery skipped", "SchemaReaper::Mailer is not loaded (ActionMailer not present?)")
|
|
88
|
+
return
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
message = @mailer.report_email(to: @config.emails, report: report_text)
|
|
92
|
+
@mail_delivery == :now ? message.deliver_now : message.deliver_later
|
|
93
|
+
rescue StandardError => e
|
|
94
|
+
log_error("email delivery failed", e)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def log_error(message, error)
|
|
98
|
+
detail = error.is_a?(Exception) ? "#{error.class}: #{error.message}" : error.to_s
|
|
99
|
+
if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
|
|
100
|
+
Rails.logger.error("[schema_reaper] #{message}: #{detail}")
|
|
101
|
+
else
|
|
102
|
+
warn("[schema_reaper] #{message}: #{detail}")
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SchemaReaper
|
|
4
|
+
# Runs a scan and delivers the report through whatever channels AlertConfig
|
|
5
|
+
# has configured. Enqueued two ways: on the schedule an app generates via
|
|
6
|
+
# `rails generate schema_reaper:install` (whenever/sidekiq-cron), and
|
|
7
|
+
# on-demand by the generated manual-trigger controller -- same job either
|
|
8
|
+
# way, so there is exactly one code path to keep correct.
|
|
9
|
+
class ScanJob < ActiveJob::Base
|
|
10
|
+
queue_as :default
|
|
11
|
+
|
|
12
|
+
# Rails' :async adapter (the default until Rails 8 unless an app sets up
|
|
13
|
+
# a real backend) runs jobs on a thread pool inside the enqueueing
|
|
14
|
+
# process. Fine in a web server; fatal in a one-shot `rake` process,
|
|
15
|
+
# which exits the moment the task returns and takes the queued job with
|
|
16
|
+
# it -- the scheduled scan would silently never run.
|
|
17
|
+
def self.in_process_queue?
|
|
18
|
+
queue_adapter.instance_of?(::ActiveJob::QueueAdapters::AsyncAdapter)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# What the scheduled cron entry (`rake schema_reaper:alert`) calls:
|
|
22
|
+
# enqueue normally, but on an in-process queue run inline instead --
|
|
23
|
+
# email included -- so the scan isn't lost when the process exits.
|
|
24
|
+
def self.run_scheduled
|
|
25
|
+
return perform_later unless in_process_queue?
|
|
26
|
+
|
|
27
|
+
puts "[schema_reaper] ActiveJob adapter is :async -- running the scan inline"
|
|
28
|
+
perform_now(deliver_mail_now: true)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# `deliver_mail_now:` is for running inline from a process about to exit
|
|
32
|
+
# (see the schema_reaper:alert rake task) -- a deliver_later there would
|
|
33
|
+
# be lost for the same reason as above.
|
|
34
|
+
def perform(deliver_mail_now: false)
|
|
35
|
+
findings = Runner.new.run
|
|
36
|
+
Notifier.new(findings, mail_delivery: deliver_mail_now ? :now : :later).deliver
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -23,4 +23,9 @@ namespace :schema_reaper do
|
|
|
23
23
|
history.record(findings)
|
|
24
24
|
pp history.trend
|
|
25
25
|
end
|
|
26
|
+
|
|
27
|
+
desc "Run the production scan+alert job (what the scheduled cron entry calls; runs inline on :async)"
|
|
28
|
+
task alert: :environment do
|
|
29
|
+
SchemaReaper::ScanJob.run_scheduled
|
|
30
|
+
end
|
|
26
31
|
end
|
data/lib/schema_reaper.rb
CHANGED
|
@@ -29,6 +29,9 @@ require_relative "schema_reaper/baseline"
|
|
|
29
29
|
require_relative "schema_reaper/history"
|
|
30
30
|
require_relative "schema_reaper/migration_generator"
|
|
31
31
|
require_relative "schema_reaper/runner"
|
|
32
|
+
require_relative "schema_reaper/alert_config"
|
|
33
|
+
require_relative "schema_reaper/notifier"
|
|
34
|
+
require_relative "schema_reaper/scan_job" if defined?(ActiveJob::Base)
|
|
32
35
|
require_relative "schema_reaper/railtie" if defined?(Rails::Railtie)
|
|
33
36
|
|
|
34
37
|
# Finds columns, indexes and tables that a Rails/ActiveRecord app no longer
|
|
@@ -36,6 +39,17 @@ require_relative "schema_reaper/railtie" if defined?(Rails::Railtie)
|
|
|
36
39
|
module SchemaReaper
|
|
37
40
|
class Error < StandardError; end
|
|
38
41
|
|
|
42
|
+
# Autoloaded, not required: Mailer picks its superclass (the host app's
|
|
43
|
+
# ApplicationMailer, when there is one) at the moment it is defined. At
|
|
44
|
+
# gem-require time -- Bundler.require, early in boot -- the app's own
|
|
45
|
+
# autoloader isn't set up yet, so ApplicationMailer can never be seen and
|
|
46
|
+
# Mailer would always fall back to ActionMailer::Base, silently losing the
|
|
47
|
+
# app's `default from:`. Deferring to first reference (when a report is
|
|
48
|
+
# actually sent, long after boot) fixes that, and still lets a worker
|
|
49
|
+
# process resolve "SchemaReaper::Mailer" by name when it deserializes the
|
|
50
|
+
# mail delivery job.
|
|
51
|
+
autoload :Mailer, File.expand_path("schema_reaper/mailer", __dir__)
|
|
52
|
+
|
|
39
53
|
REPORTERS = {
|
|
40
54
|
"table" => Reporters::Table,
|
|
41
55
|
"json" => Reporters::Json,
|
data/schema_reaper.gemspec
CHANGED
|
@@ -37,7 +37,7 @@ Gem::Specification.new do |spec|
|
|
|
37
37
|
spec.files = Dir.chdir(__dir__) do
|
|
38
38
|
`git ls-files -z`.split("\x0").reject do |f|
|
|
39
39
|
(File.expand_path(f) == __FILE__) ||
|
|
40
|
-
f.start_with?(*%w[bin/ test/ spec/ features/ .git .github Gemfile])
|
|
40
|
+
f.start_with?(*%w[bin/ test/ spec/ features/ docs/ .git .github Gemfile])
|
|
41
41
|
end
|
|
42
42
|
end
|
|
43
43
|
spec.bindir = "exe"
|
|
@@ -47,6 +47,10 @@ Gem::Specification.new do |spec|
|
|
|
47
47
|
spec.add_dependency "prism", ">= 0.19", "< 2.0"
|
|
48
48
|
spec.add_dependency "thor", "~> 1.3"
|
|
49
49
|
|
|
50
|
+
spec.add_development_dependency "actionmailer", ">= 6.1", "< 9.0"
|
|
51
|
+
spec.add_development_dependency "activejob", ">= 6.1", "< 9.0"
|
|
50
52
|
spec.add_development_dependency "activerecord", ">= 6.1", "< 9.0"
|
|
51
53
|
spec.add_development_dependency "pg", "~> 1.5"
|
|
54
|
+
spec.add_development_dependency "railties", ">= 6.1", "< 9.0"
|
|
55
|
+
spec.add_development_dependency "simplecov", "~> 0.22"
|
|
52
56
|
end
|