schema_reaper 2.0.0 → 2.1.0

data/lib/generators/schema_reaper/install/install_generator.rb

@@ -48,7 +48,7 @@ module SchemaReaper
 
         say_status :warning,
                    "schema_reaper is Gemfile-scoped to group: :development. Most production deploy " \
-                   "pipelines strip dev/test groups (`bundle install --without development test`), so " \
+                   "pipelines strip dev/test groups (`BUNDLE_WITHOUT=development:test`), so " \
                    "the scheduled scan will not run in production until you remove that restriction.",
                    :red
       end
@@ -57,7 +57,8 @@ module SchemaReaper
         return if mailer_from_configured?
 
         say_status :warning,
-                   "no ApplicationMailer default `from:` detected. SchemaReaper::Mailer falls back to a " \
+                   "no ApplicationMailer default `from:` detected (nor config.action_mailer.default_options). " \
+                   "SchemaReaper::Mailer falls back to a " \
                    "placeholder sender in that case, which most real SMTP relays reject or spam-flag -- " \
                    "email reports would silently fail to arrive. If you plan to use the email channel " \
                    "(config.emails in the initializer this generator just wrote), set " \
@@ -152,7 +153,7 @@ module SchemaReaper
       # `gem "schema_reaper", group: :development` (or the equivalent
       # `group :development do ... end` block form) in the app's own Gemfile
       # -- see README's current install snippet. A dev-scoped gem is absent
-      # from `bundle install --without development test`, which most
+      # from `BUNDLE_WITHOUT=development:test` installs, which most
       # production deploy pipelines run, so the scheduled scan silently
       # never runs.
       #
@@ -171,15 +172,14 @@ module SchemaReaper
         false # a Gemfile we can't parse shouldn't block the rest of the generator
       end
 
-      # Checked against ApplicationMailer specifically, not SchemaReaper::Mailer
-      # -- at generate time the app is already booted (ActionMailer, and
-      # ApplicationMailer if the app has one, are loaded), so this reads the
-      # same `default_params[:from]` chain SchemaReaper::Mailer itself will
-      # inherit from once it's loaded.
+      # Checked against the class SchemaReaper::Mailer will inherit from --
+      # ApplicationMailer when the app has one, ActionMailer::Base otherwise
+      # (where `config.action_mailer.default_options = { from: ... }` lands).
+      # At generate time the app is already booted, so this reads the same
+      # `default_params[:from]` chain Mailer itself will see.
       def mailer_from_configured?
-        return false unless defined?(::ApplicationMailer) # no mailer to inherit from at all
-
-        ::ApplicationMailer.default_params[:from].present?
+        parent = defined?(::ApplicationMailer) ? ::ApplicationMailer : ::ActionMailer::Base
+        parent.default_params[:from].present?
       rescue StandardError
         true # can't determine -- don't nag over something we're unsure about
       end

data/lib/generators/schema_reaper/install/templates/initializer.rb.tt

@@ -15,4 +15,12 @@
 SchemaReaper::AlertConfig.configure do |config|
   # config.emails = %w[dev1@example.com dev2@example.com]
   # config.webhook_url = "https://hooks.slack.com/services/T000/B000/XXXX"
+
+  # :changes sends a report only when findings appeared or were resolved
+  # since the last one (needs Rails.cache); :always sends every run.
+  # config.alert_on = :changes
+
+  # :auto uses Slack Block Kit for hooks.slack.com URLs and a plain
+  # {"text": <markdown>} body for anything else. Force with :slack or :text.
+  # config.webhook_format = :auto
 end

data/lib/schema_reaper/alert_config.rb

@@ -30,11 +30,31 @@ module SchemaReaper
       end
     end
 
-    attr_accessor :emails, :webhook_url
+    # alert_on:       :always  -- send every scheduled report (the 2.0 behaviour)
+    #                 :changes -- send only when findings appeared or were
+    #                             resolved since the last report
+    # webhook_format: :auto    -- Slack Block Kit for hooks.slack.com, plain
+    #                             {"text": <markdown>} for anything else
+    #                 :slack / :text to force one
+    attr_accessor :emails, :webhook_url, :alert_on, :webhook_format
 
     def initialize
       @emails = []
       @webhook_url = nil
+      @alert_on = :always
+      @webhook_format = :auto
+    end
+
+    def changes_only?
+      alert_on.to_s == "changes"
+    end
+
+    def slack_blocks?
+      case webhook_format.to_s
+      when "slack" then true
+      when "text" then false
+      else webhook_url.to_s.include?("hooks.slack.com")
+      end
     end
 
     def emails?

data/lib/schema_reaper/alert_state.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  # Remembers which finding ids the last delivered report contained, so the
+  # next one can say what changed -- and, with alert_on: :changes, stay quiet
+  # when nothing did. Kept in Rails.cache (the manual trigger's cooldown
+  # already relies on it) because production filesystems are often
+  # ephemeral. Without a cache every report is treated as the first.
+  class AlertState
+    KEY = "schema_reaper:last_alert_ids"
+
+    def initialize(cache: default_cache)
+      @cache = cache
+    end
+
+    # @return [Array<String>, nil] nil when no earlier report is known
+    def previous_ids
+      @cache&.read(KEY)
+    rescue StandardError
+      nil
+    end
+
+    def remember(ids)
+      @cache&.write(KEY, ids.sort)
+    rescue StandardError
+      nil
+    end
+
+    private
+
+    def default_cache
+      defined?(Rails) && Rails.respond_to?(:cache) ? Rails.cache : nil
+    end
+  end
+end

data/lib/schema_reaper/analyzers/always_null_column.rb

@@ -20,6 +20,7 @@ module SchemaReaper
         table.columns.filter_map do |col|
           next unless col.always_null?
           next if config.always_keep_columns.include?(col.name)
+          next if ignored_column?(col.name)
           next if gem_reserved?(table.name, col.name)
 
           finding(
@@ -27,17 +28,23 @@ module SchemaReaper
             table: table.name,
             column: col.name,
             severity: :high,
-            confidence: 0.85,
+            confidence: sampled_stats?(table) ? 0.7 : 0.85,
             bytes_per_row: col.bytes,
-            evidence: [
-              "pg_stats.null_frac = 1.0 across ~#{table.row_count} row(s)",
-              "column carries no data"
-            ],
+            evidence: evidence_for(table),
             suggested_fix: "verify with `SELECT count(#{col.name}) FROM #{table.name}` " \
                            "then stage a removal"
           )
         end
       end
+
+      def evidence_for(table)
+        ev = ["pg_stats.null_frac = 1.0 across ~#{table.row_count} row(s)"]
+        ev << if sampled_stats?(table)
+                "estimated from a #{STATS_SAMPLE_ROWS}-row ANALYZE sample -- a few non-null rows could be missed"
+              else
+                "column carries no data"
+              end
+      end
     end
   end
 end

data/lib/schema_reaper/analyzers/base.rb

@@ -11,8 +11,14 @@ module SchemaReaper
     #   gem_columns      - Hash{table_name => Set<column_name>} reserved by gems
     #   gem_owned_tables - Set<table_name> created and owned outright by a gem
     #   config           - Config
-    Context = Struct.new(:schema, :used_tokens, :runtime, :gem_columns, :gem_owned_tables, :config,
+    #   scan             - Static::Result (optional; per-model detail
+    #                      for column_scope: model)
+    Context = Struct.new(:schema, :used_tokens, :runtime, :gem_columns, :gem_owned_tables, :config, :scan,
                          keyword_init: true) do
+      def scan
+        self[:scan] || Static::Result.from_tokens(used_tokens || [])
+      end
+
       def runtime
         self[:runtime] || Runtime::Report.empty
       end
@@ -28,6 +34,11 @@ module SchemaReaper
 
     # Shared plumbing for analyzers: schema access and token lookup helpers.
     class Base
+      # ANALYZE reads a sample of 300 x default_statistics_target rows (30,000
+      # at the default of 100). Above that, pg_stats' null_frac and n_distinct
+      # are estimates: a value held by a handful of rows can be missed.
+      STATS_SAMPLE_ROWS = 30_000
+
       def self.type
         name.split("::").last.gsub(/([a-z])([A-Z])/, '\1_\2').downcase.to_sym
       end
@@ -61,8 +72,61 @@ module SchemaReaper
         ctx.used_tokens.include?(token.to_s.downcase)
       end
 
+      # Whether code references `column` on `table`. With column_scope: model,
+      # a name seen only in model files counts just for the tables those
+      # models are, or mention (an association, a class name), so Order's
+      # `status` no longer hides a dead `status` on users.
+      def column_used?(table, column)
+        return used?(column) unless config.column_scope == :model
+
+        name = column.to_s.downcase
+        scan = ctx.scan
+        return true if scan.shared.include?(name)
+
+        forms = table_forms(table)
+        scan.models.any? do |model|
+          model.tokens.include?(name) && (model.tables.include?(table) || forms.any? { |f| model.tokens.include?(f) })
+        end
+      end
+
+      def table_forms(table)
+        bare = table.to_s.split(".").last
+        singular = Inflect.singularize(bare)
+        [bare, singular, singular.delete("_")].uniq
+      end
+
+      # Every column of a table a gem owns outright is the gem's business.
       def gem_reserved?(table, column)
-        ctx.gem_columns.fetch(table, []).include?(column)
+        gem_owned_table?(table) || ctx.gem_columns.fetch(table, []).include?(column)
+      end
+
+      def sampled_stats?(table)
+        table.row_count.to_i > STATS_SAMPLE_ROWS
+      end
+
+      # The `ignore.columns` list (names or /regex/) applies to every
+      # column-level analyzer, not only dead_column.
+      def ignored_column?(name)
+        config.ignored_column?(name)
+      end
+
+      # The removal command for an index. DROP INDEX CONCURRENTLY is refused
+      # on a partitioned index ("cannot drop partitioned index ...
+      # concurrently"), and its per-partition indexes can't be dropped on
+      # their own while attached, so the only path is a plain drop, which
+      # takes an ACCESS EXCLUSIVE lock on the parent and every partition.
+      def remove_index_fix(table, index)
+        base = "remove_index #{sym(table.name)}, name: :#{index.name}"
+        return "#{base}  (use algorithm: :concurrently in production)" unless index.partitioned
+
+        "#{base}  (partitioned index: CONCURRENTLY is not possible -- this locks the table " \
+          "and all partitions while it runs, so schedule it for a quiet window)"
+      end
+
+      # A table name as a Ruby symbol literal for suggested fixes: :users, or
+      # :"billing.invoices" for a table outside the public schema.
+      def sym(name)
+        name.to_sym.inspect
       end
 
       def gem_owned_table?(table)

data/lib/schema_reaper/analyzers/dead_column.rb

@@ -4,12 +4,12 @@ module SchemaReaper
   module Analyzers
     # Flags columns present in the schema but never referenced in code. When a
     # runtime usage log is supplied, its signal is fused in: a column unseen in
-    # BOTH code and >= 14 observed days of runtime reaches high confidence.
+    # BOTH code and at least `min_age_days` (default 14) observed days of
+    # runtime reaches high confidence.
     class DeadColumn < Base
       Registry.register(self)
 
-      STATIC_ONLY_CAP  = 0.6
-      RUNTIME_MIN_DAYS = 14
+      STATIC_ONLY_CAP = 0.6
 
       def call
         schema.tables.reject { |t| config.ignore_tables.include?(t.name) }
@@ -21,7 +21,7 @@ module SchemaReaper
       def dead_in(table)
         table.columns.filter_map do |col|
           next if keep?(table, col)
-          next if used?(col.name)
+          next if column_used?(table.name, col.name)
           next if runtime.read?(table.name, col.name)
 
           finding(
@@ -33,7 +33,7 @@ module SchemaReaper
             bytes_per_row: col.bytes,
             evidence: evidence_for(table, col),
             suggested_fix: "Stage removal: `self.ignored_columns += %w[#{col.name}]` on the " \
-                           "model, deploy, then `remove_column :#{table.name}, :#{col.name}`."
+                           "model, deploy, then `remove_column #{sym(table.name)}, :#{col.name}`."
           )
         end
       end
@@ -48,7 +48,7 @@ module SchemaReaper
       end
 
       def confidence_for(col)
-        if runtime.present? && runtime.observed_days >= RUNTIME_MIN_DAYS
+        if runtime.present? && runtime.observed_days >= min_runtime_days
           col.null ? 0.9 : 0.8
         else
           base = col.null ? 0.5 : 0.4
@@ -56,6 +56,12 @@ module SchemaReaper
         end
       end
 
+      # How many days of runtime data it takes before "never read at runtime"
+      # is trusted. A config built without the key (nil) keeps the default.
+      def min_runtime_days
+        config.min_age_days || Config::DEFAULTS["min_age_days"]
+      end
+
       def evidence_for(table, col)
         ev = ["no `#{col.name}` reference found in scanned code"]
         ev << "column is nullable" if col.null

data/lib/schema_reaper/analyzers/dead_table.rb

@@ -24,13 +24,13 @@ module SchemaReaper
 
       def ignored?(table)
         config.ignore_tables.include?(table.name) ||
-          table.name.start_with?("active_storage_", "action_text_", "action_mailbox_") ||
+          table.name.split(".").last.start_with?("active_storage_", "action_text_", "action_mailbox_") ||
           gem_owned_table?(table.name)
       end
 
       def dead(table)
         return if referenced?(table)
-        return if runtime.accessed.any? { |k| k.start_with?("#{table.name}.") }
+        return if runtime.table_accessed?(table.name)
 
         finding(
           type: :dead_table,
@@ -40,17 +40,21 @@ module SchemaReaper
           confidence: confidence_for(table),
           bytes_per_row: table.columns.sum(&:bytes),
           evidence: evidence_for(table),
-          suggested_fix: "confirm no external consumer, then `drop_table :#{table.name}`"
+          suggested_fix: "confirm no external consumer, then `drop_table #{sym(table.name)}`"
         )
       end
 
       # Match the table name and its singular/camelized model form.
+      # A table outside public ("billing.invoices") is still referenced by its
+      # bare name in code -- `self.table_name = "billing.invoices"` tokenises
+      # to billing + invoices, and the model is Invoice.
       def referenced?(table)
-        singular = singularize(table.name)
-        forms = [table.name, singular, camelize(table.name), camelize(singular)]
+        bare = table.name.split(".").last
+        singular = singularize(bare)
+        forms = [bare, singular, camelize(bare), camelize(singular)]
         return true if forms.any? { |form| used?(form) }
 
-        embedded_in_identifier?(table.name)
+        embedded_in_identifier?(bare)
       end
 
       def singularize(name)

data/lib/schema_reaper/analyzers/duplicate_foreign_key.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  module Analyzers
+    # Two FOREIGN KEY constraints on the same columns pointing at the same
+    # key of the same table, with the same ON UPDATE / ON DELETE, MATCH and
+    # deferrability -- only then is either one redundant. (Same columns but a
+    # different ON DELETE is not a duplicate: dropping one changes what a
+    # parent delete does.) Usually a hand-written add_foreign_key layered on a
+    # `t.references ..., foreign_key: true` that already made one. Each copy is
+    # checked on every insert/update of the child and every delete of the
+    # parent, so the extra one is pure overhead.
+    class DuplicateForeignKey < Base
+      Registry.register(self)
+
+      def call
+        schema.tables.reject { |t| config.ignore_tables.include?(t.name) }
+              .flat_map { |t| duplicates_in(t) }
+      end
+
+      private
+
+      def duplicates_in(table)
+        table.foreign_key_constraints
+             .group_by { |fk| [fk.columns, fk.references, fk.behaviour] }
+             .values.select { |group| group.size > 1 }
+             .flat_map { |group| redundant(table, group) }
+      end
+
+      # Keep a validated one (a NOT VALID copy doesn't check existing rows),
+      # then the alphabetically first name -- Rails' fk_rails_<hash> sorts
+      # ahead of hand-named ones. Report the rest.
+      def redundant(table, group)
+        keep, *extra = group.sort_by { |fk| [fk.validated == false ? 1 : 0, fk.name] }
+        extra.map do |fk|
+          finding(
+            type: :duplicate_foreign_key,
+            table: table.name,
+            column: fk.columns.join(","),
+            index: fk.name,
+            severity: :low,
+            confidence: 0.9,
+            bytes_per_row: 0,
+            evidence: [
+              "#{fk.name} and #{keep.name} both constrain (#{fk.columns.join(", ")}) -> #{fk.references}"
+            ],
+            suggested_fix: "remove_foreign_key #{sym(table.name)}, name: :#{fk.name}"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/schema_reaper/analyzers/duplicate_index.rb

@@ -40,7 +40,7 @@ module SchemaReaper
           confidence: 0.8,
           bytes_per_row: 0,
           evidence: [evidence_for(index, covering)],
-          suggested_fix: "remove_index :#{table.name}, name: :#{index.name}"
+          suggested_fix: remove_index_fix(table, index)
         )
       end
 

data/lib/schema_reaper/analyzers/missing_fk_index.rb

@@ -32,6 +32,8 @@ module SchemaReaper
 
       def missing_in(table)
         fk_columns(table).filter_map do |col|
+          next if ignored_column?(col)
+
           type_col = polymorphic_type_for(table, col)
 
           # A bare index on the id alone doesn't cover a polymorphic pair --
@@ -56,7 +58,7 @@ module SchemaReaper
       end
 
       def finding_for(table, col, type_col, declared:)
-        referent = declared ? nil : referent_table(col)
+        referent = declared ? nil : referent_table(col, table.name)
         confidence = confidence_for(declared, referent)
 
         finding(
@@ -97,9 +99,9 @@ module SchemaReaper
       end
 
       def fix_for(table, col, type_col)
-        return "add_index :#{table.name}, :#{col}" unless type_col
+        return "add_index #{sym(table.name)}, :#{col}" unless type_col
 
-        "add_index :#{table.name}, %i[#{type_col} #{col}]"
+        "add_index #{sym(table.name)}, %i[#{type_col} #{col}]"
       end
 
       def fk_columns(table)
@@ -109,7 +111,9 @@ module SchemaReaper
       # A table the column plausibly points at: account_id -> accounts,
       # company_id -> companies, address_id -> addresses. Absence is the signal
       # that a *_id name is not really a foreign key.
-      def referent_table(col)
+      # A table outside public looks in its own schema first:
+      # billing.invoices.account_id -> billing.accounts, then accounts.
+      def referent_table(col, from_table)
         return nil unless col.end_with?("_id")
 
         base = col[0..-4]
@@ -117,6 +121,8 @@ module SchemaReaper
 
         candidates = [base, "#{base}s", "#{base}es"]
         candidates << "#{base[0..-2]}ies" if base.end_with?("y")
+        ns = from_table.include?(".") ? from_table.split(".").first : nil
+        candidates = candidates.map { |c| "#{ns}.#{c}" } + candidates if ns
         candidates.find { |name| table_names.include?(name) }
       end
 

data/lib/schema_reaper/analyzers/sequence_exhaustion.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  module Analyzers
+    # A serial / IDENTITY sequence that has used up a large share of the values
+    # its column can hold. When it runs out, every INSERT into the table fails.
+    # The classic case is a pre-Rails-5.1 `integer` primary key, capped at
+    # 2,147,483,647 even when its sequence is bigint.
+    #
+    # Not dead weight -- a health check, like missing_fk_index -- but it is
+    # read straight from the schema scan the other analyzers already need.
+    class SequenceExhaustion < Base
+      Registry.register(self)
+
+      WARN_AT = 0.5
+      HIGH_AT = 0.75
+
+      def call
+        schema.sequences.filter_map do |seq|
+          next if config.ignore_tables.include?(seq.table)
+          next if seq.used_fraction < WARN_AT
+
+          finding_for(seq)
+        end
+      end
+
+      private
+
+      def finding_for(seq)
+        pct = (seq.used_fraction * 100).round(1)
+        finding(
+          type: :sequence_exhaustion,
+          table: seq.table,
+          column: seq.column,
+          severity: seq.used_fraction >= HIGH_AT ? :high : :medium,
+          confidence: 0.95, # read from the database, not inferred
+          bytes_per_row: 0,
+          evidence: [
+            "#{seq.name} is at #{seq.last_value} of #{seq.max_value} (#{pct}%) " \
+            "for #{seq.column_type} column #{seq.column}",
+            "INSERTs into #{seq.table} fail once it is exhausted"
+          ],
+          suggested_fix: fix_for(seq)
+        )
+      end
+
+      def fix_for(seq)
+        return "the sequence itself is near its MAXVALUE -- raise it with ALTER SEQUENCE" if seq.column_type == "bigint"
+
+        "change_column #{sym(seq.table)}, :#{seq.column}, :bigint  " \
+          "(rewrites the table under an exclusive lock -- plan for it on a large table)"
+      end
+    end
+  end
+end

data/lib/schema_reaper/analyzers/single_value_column.rb

@@ -24,6 +24,7 @@ module SchemaReaper
           next unless col.single_value?
           next if table.primary_key?(col.name)
           next if config.always_keep_columns.include?(col.name)
+          next if ignored_column?(col.name)
           next if gem_reserved?(table.name, col.name)
 
           finding(
@@ -31,16 +32,27 @@ module SchemaReaper
             table: table.name,
             column: col.name,
             severity: :low,
-            confidence: 0.6,
+            confidence: sampled_stats?(table) ? 0.5 : 0.6,
             bytes_per_row: col.bytes,
-            evidence: [
-              "pg_stats.n_distinct = 1 across ~#{table.row_count} row(s)",
-              "column has the same value in every row"
-            ],
+            evidence: evidence_for(table, col),
             suggested_fix: "confirm the value is not a meaningful default before removing"
           )
         end
       end
+
+      # n_distinct counts non-null values only, so "one value" and "the same
+      # value in every row" are different claims when some rows are NULL.
+      def evidence_for(table, col)
+        nulls = col.null_fraction.to_f
+        ev = ["pg_stats.n_distinct = 1 across ~#{table.row_count} row(s)"]
+        ev << if nulls.positive?
+                "one distinct non-null value; #{(nulls * 100).round(1)}% of rows are NULL"
+              else
+                "column has the same value in every row"
+              end
+        ev << "estimated from a #{STATS_SAMPLE_ROWS}-row ANALYZE sample" if sampled_stats?(table)
+        ev
+      end
     end
   end
 end

data/lib/schema_reaper/analyzers/unused_index.rb

@@ -34,15 +34,20 @@ module SchemaReaper
             severity: :medium,
             confidence: 0.55,
             bytes_per_row: 0,
-            evidence: [
-              "pg_stat_user_indexes.idx_scan = 0 for #{ix.name} (#{ix.columns.join(", ")})",
-              "confirm stats have not been reset recently before dropping"
-            ],
-            suggested_fix: "remove_index :#{table.name}, name: :#{ix.name}  " \
-                           "(use algorithm: :concurrently in production)"
+            evidence: evidence_for(ix),
+            suggested_fix: remove_index_fix(table, ix)
           )
         end
       end
+
+      def evidence_for(index)
+        [
+          "pg_stat_user_indexes.idx_scan = 0 for #{index.name} (#{index.columns.join(", ")})",
+          "confirm stats have not been reset recently before dropping",
+          "idx_scan is per server: if read replicas serve queries, check " \
+          "pg_stat_user_indexes on each replica too -- an index used only there looks unused here"
+        ]
+      end
     end
   end
 end