schema_reaper 1.0.0

data/lib/schema_reaper/migration_generator.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module SchemaReaper
+  # Emits a two-step, reversible migration pair for a dead column.
+  class MigrationGenerator
+    def initialize(table:, column:, dir: "db/migrate")
+      @table = table
+      @column = column
+      @dir = dir
+    end
+
+    def call
+      FileUtils.mkdir_p(@dir)
+      [stage_one, stage_two]
+    end
+
+    private
+
+    def stage_one
+      write "ignore_#{@table}_#{@column}", <<~RUBY
+        # frozen_string_literal: true
+
+        # STEP 1 of 2. Deploy this alone and let it soak. It only tells
+        # ActiveRecord to stop selecting the column; nothing is dropped.
+        class Ignore#{camel}Column < ActiveRecord::Migration[7.1]
+          def up
+            say "Add `self.ignored_columns += %w[#{@column}]` to the #{model} model, " \
+                "then deploy STEP 2 after a soak period."
+          end
+
+          def down; end
+        end
+      RUBY
+    end
+
+    def stage_two
+      write "drop_#{@table}_#{@column}", <<~RUBY
+        # frozen_string_literal: true
+
+        # STEP 2 of 2. Run only after STEP 1 has been deployed and verified.
+        class Drop#{camel}Column < ActiveRecord::Migration[7.1]
+          def up
+            remove_column :#{@table}, :#{@column}
+          end
+
+          def down
+            raise ActiveRecord::IrreversibleMigration,
+                  "recreate :#{@column} on :#{@table} manually if you need it back"
+          end
+        end
+      RUBY
+    end
+
+    def write(slug, body)
+      @seq = (@seq || -1) + 1
+      ts = (Time.now + @seq).strftime("%Y%m%d%H%M%S")
+      path = File.join(@dir, "#{ts}_#{slug}.rb")
+      File.write(path, body)
+      path
+    end
+
+    def camel = "#{@table}_#{@column}".split("_").map(&:capitalize).join
+    def model = @table.split("_").map(&:capitalize).join.sub(/s$/, "")
+  end
+end

data/lib/schema_reaper/railtie.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+
+module SchemaReaper
+  # Loads rake tasks and, when enabled, installs the runtime tracker.
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.expand_path("tasks/schema_reaper.rake", __dir__)
+    end
+
+    initializer "schema_reaper.runtime_tracker" do
+      next unless ENV["SCHEMA_REAPER_TRACK"] == "1"
+
+      config = SchemaReaper::Config.load
+      store = SchemaReaper::Runtime::Store.new(path: config.runtime_log)
+      rate = (ENV["SCHEMA_REAPER_SAMPLE"] || "0.05").to_f
+      SchemaReaper::Runtime::Tracker.install!(store: store, sample_rate: rate)
+    end
+  end
+end

data/lib/schema_reaper/reporters/bytes.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  module Reporters
+    # Formats a byte count as a human-readable size (e.g. "3.4 MB").
+    module Bytes
+      UNITS = %w[B KB MB GB TB].freeze
+
+      module_function
+
+      def human(n)
+        n = n.to_f
+        idx = 0
+        while n >= 1024 && idx < UNITS.length - 1
+          n /= 1024
+          idx += 1
+        end
+        format("%.1f %s", n, UNITS[idx])
+      end
+    end
+  end
+end

data/lib/schema_reaper/reporters/json.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module SchemaReaper
+  module Reporters
+    # Machine-readable output for CI and downstream tooling.
+    class Json
+      def initialize(findings, io: $stdout)
+        @findings = findings
+        @io = io
+      end
+
+      def render
+        @io.puts JSON.pretty_generate(payload)
+      end
+
+      # Also used by History to snapshot a run.
+      def payload
+        {
+          version: SchemaReaper::VERSION,
+          generated_at: Time.now.utc.iso8601,
+          count: @findings.size,
+          reclaimable_bytes: @findings.sum(&:reclaimable_bytes),
+          findings: @findings.map(&:to_h)
+        }
+      end
+    end
+  end
+end

data/lib/schema_reaper/reporters/markdown.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative "bytes"
+
+module SchemaReaper
+  module Reporters
+    # GitHub-flavoured Markdown, suitable for a PR comment or job summary.
+    class Markdown
+      def initialize(findings, io: $stdout)
+        @findings = findings
+        @io = io
+      end
+
+      def render
+        @io.puts "## schema_reaper"
+        if @findings.empty?
+          @io.puts "\nNo findings. Schema is lean. :sparkles:"
+          return
+        end
+
+        @io.puts "\n#{@findings.size} finding(s), " \
+                 "~**#{Bytes.human(total)}** reclaimable.\n\n"
+        @io.puts "| Severity | Confidence | Type | Target | Reclaims | Fix |"
+        @io.puts "|---|---|---|---|---|---|"
+        rows.each { |r| @io.puts r }
+      end
+
+      private
+
+      def rows
+        @findings.sort_by { |f| -f.confidence }.map do |f|
+          target = [f.table, f.column, f.index].compact.join("`.`")
+          "| #{f.severity} | #{(f.confidence * 100).round}% | `#{f.type}` | " \
+            "`#{target}` | #{Bytes.human(f.reclaimable_bytes)} | #{f.suggested_fix} |"
+        end
+      end
+
+      def total = @findings.sum(&:reclaimable_bytes)
+    end
+  end
+end

data/lib/schema_reaper/reporters/sarif.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SchemaReaper
+  module Reporters
+    # SARIF 2.1.0 so findings show up in GitHub code scanning.
+    class Sarif
+      LEVEL = { high: "error", medium: "warning", low: "note" }.freeze
+
+      def initialize(findings, io: $stdout)
+        @findings = findings
+        @io = io
+      end
+
+      def render
+        @io.puts JSON.pretty_generate(document)
+      end
+
+      private
+
+      def document
+        {
+          "$schema" => "https://json.schemastore.org/sarif-2.1.0.json",
+          "version" => "2.1.0",
+          "runs" => [{
+            "tool" => { "driver" => {
+              "name" => "schema_reaper",
+              "version" => SchemaReaper::VERSION,
+              "informationUri" => "https://github.com/aksshatt/schema_reaper",
+              "rules" => rules
+            } },
+            "results" => results
+          }]
+        }
+      end
+
+      def rules
+        @findings.map(&:type).uniq.map do |type|
+          { "id" => type.to_s, "name" => camelize(type.to_s),
+            "shortDescription" => { "text" => "schema dead-weight: #{type}" } }
+        end
+      end
+
+      def results
+        @findings.map do |f|
+          {
+            "ruleId" => f.type.to_s,
+            "level" => LEVEL.fetch(f.severity, "note"),
+            "message" => { "text" => "#{f.id}: #{f.evidence.join("; ")}. Fix: #{f.suggested_fix}" },
+            "properties" => {
+              "confidence" => f.confidence,
+              "reclaimableBytes" => f.reclaimable_bytes
+            },
+            "locations" => [{
+              "physicalLocation" => {
+                "artifactLocation" => { "uri" => "db/schema.rb" }
+              }
+            }]
+          }
+        end
+      end
+
+      def camelize(str) = str.split("_").map(&:capitalize).join
+    end
+  end
+end

data/lib/schema_reaper/reporters/table.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative "bytes"
+
+module SchemaReaper
+  module Reporters
+    # Human-readable terminal output, sorted by confidence then severity.
+    class Table
+      SEV_ORDER = { high: 0, medium: 1, low: 2 }.freeze
+
+      def initialize(findings, io: $stdout)
+        @findings = findings
+        @io = io
+      end
+
+      def render
+        if @findings.empty?
+          @io.puts "schema_reaper: no findings. Schema is lean."
+          return
+        end
+
+        sorted.each { |f| render_finding(f) }
+        @io.puts
+        @io.puts "#{@findings.size} finding(s). " \
+                 "~#{Bytes.human(total_reclaimable)} reclaimable."
+      end
+
+      private
+
+      def render_finding(f)
+        target = [f.table, f.column, f.index].compact.join(".")
+        @io.puts format("[%-6s %3d%%] %-14s %s",
+                        f.severity, (f.confidence * 100).round, f.type, target)
+        f.evidence.each { |e| @io.puts "        - #{e}" }
+        @io.puts "        fix: #{f.suggested_fix}"
+      end
+
+      def sorted
+        @findings.sort_by { |f| [-f.confidence, SEV_ORDER.fetch(f.severity, 9)] }
+      end
+
+      def total_reclaimable = @findings.sum(&:reclaimable_bytes)
+    end
+  end
+end

data/lib/schema_reaper/runner.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  # Ties introspection + scanning + runtime data + analyzers together.
+  class Runner
+    def initialize(config: Config.load, root: Dir.pwd, introspector: nil, runtime: nil)
+      @config = config
+      @root = root
+      @introspector = introspector
+      @runtime = runtime
+      load_plugins
+    end
+
+    def run
+      db = schema
+      ctx = Analyzers::Context.new(
+        schema: db,
+        used_tokens: Static::Scanner.new(@config, root: @root).call,
+        runtime: runtime_report,
+        gem_columns: gem_columns(db),
+        config: @config
+      )
+
+      findings = Analyzers::Registry.all.flat_map { |klass| klass.new(ctx).call }
+      dedupe(findings).sort_by { |f| [-f.confidence, f.id] }
+    end
+
+    private
+
+    # When a whole table is dead, its per-column and per-index findings are
+    # noise -- keep only the table-level finding for that table.
+    def dedupe(findings)
+      dead_tables = findings.select { |f| f.type == :dead_table }.to_set(&:table)
+      findings.reject { |f| f.type != :dead_table && dead_tables.include?(f.table) }
+    end
+
+    def schema
+      return @introspector.call if @introspector
+
+      Introspect::Postgres.new(@config.database_url).call
+    end
+
+    def runtime_report
+      return @runtime if @runtime
+
+      Runtime::Report.load(@config.runtime_log)
+    end
+
+    def gem_columns(db)
+      return {} unless @config.gem_awareness?
+
+      GemAwareness.reserved_columns(
+        installed: GemAwareness.installed_gems,
+        tables: db.tables
+      )
+    end
+
+    def load_plugins
+      @config.require_paths.each do |path|
+        require(File.expand_path(path, @root))
+      end
+    end
+  end
+end

data/lib/schema_reaper/runtime.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "json"
+require "set"
+require "fileutils"
+require "time"
+
+module SchemaReaper
+  # Optional production signal: which columns are actually read/written at
+  # runtime. Fused with the static scan to raise (or confirm) confidence.
+  module Runtime
+    # Aggregated view of a usage log, consumed by analyzers.
+    Report = Struct.new(:accessed, :observed_days, keyword_init: true) do
+      def self.empty = new(accessed: Set.new, observed_days: 0)
+
+      def self.load(path)
+        return empty unless path && File.exist?(path)
+
+        seen = Set.new
+        first = last = nil
+        File.foreach(path) do |line|
+          row = JSON.parse(line)
+          seen << row["key"]
+          ts = row["at"]
+          first ||= ts
+          last = ts
+        rescue JSON::ParserError
+          next
+        end
+        new(accessed: seen, observed_days: day_span(first, last))
+      end
+
+      def self.day_span(first, last)
+        return 0 unless first && last
+
+        ((Time.parse(last) - Time.parse(first)) / 86_400).ceil
+      end
+
+      def empty? = accessed.empty?
+      def present? = !empty?
+      def read?(table, column) = accessed.include?("#{table}.#{column}")
+    end
+
+    # Buffered writer for the usage log. Thread-safe append of JSON lines.
+    class Store
+      def initialize(path:, flush_every: 200)
+        @path = path
+        @flush_every = flush_every
+        @buffer = []
+        @mutex = Mutex.new
+        FileUtils.mkdir_p(File.dirname(path))
+      end
+
+      def record(table, column)
+        @mutex.synchronize do
+          @buffer << %({"key":"#{table}.#{column}","at":"#{Time.now.utc.iso8601}"}\n)
+          flush_locked if @buffer.size >= @flush_every
+        end
+      end
+
+      def flush
+        @mutex.synchronize { flush_locked }
+      end
+
+      private
+
+      def flush_locked
+        return if @buffer.empty?
+
+        File.open(@path, "a") { |f| f.write(@buffer.join) }
+        @buffer.clear
+      end
+    end
+
+    # Patches ActiveRecord attribute access to feed a Store. Sampling keeps
+    # production overhead negligible.
+    module Tracker
+      class << self
+        attr_accessor :store, :sample_rate
+
+        def install!(store:, sample_rate: 0.05)
+          return if @installed
+
+          self.store = store
+          self.sample_rate = sample_rate
+          require "active_record"
+          ActiveRecord::Base.prepend(Hook)
+          at_exit { store.flush }
+          @installed = true
+        end
+      end
+
+      module Hook
+        def _read_attribute(name, *)
+          SchemaReaper::Runtime::Tracker.note(self.class, name)
+          super
+        end
+      end
+
+      def self.note(klass, name)
+        return unless store
+        return unless rand < sample_rate
+        return unless klass.respond_to?(:table_name) && klass.table_name
+
+        store.record(klass.table_name, name.to_s)
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/schema_reaper/schema.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  # Plain-data representation of the database, produced by an introspector.
+  # No ActiveRecord objects leak past this boundary.
+  Column = Struct.new(
+    :name, :sql_type, :null, :default, :bytes,
+    :distinct_values, :null_fraction,
+    keyword_init: true
+  ) do
+    # true when every row holds NULL (pg_stats null_frac == 1)
+    def always_null? = null_fraction && null_fraction >= 1.0
+
+    # true when the column holds exactly one distinct non-null value
+    def single_value? = distinct_values == 1
+  end
+
+  Index = Struct.new(:name, :columns, :unique, :primary, :scans, keyword_init: true) do
+    def covers?(other) = columns.first(other.columns.length) == other.columns
+  end
+
+  Table = Struct.new(
+    :name, :columns, :indexes, :primary_key, :foreign_keys, :row_count,
+    keyword_init: true
+  ) do
+    def column_names = columns.map(&:name)
+    def column(name) = columns.find { |c| c.name == name }
+    def index_on(cols) = indexes.find { |i| i.columns == Array(cols) }
+  end
+
+  DatabaseSchema = Struct.new(:tables, keyword_init: true) do
+    def table(name) = tables.find { |t| t.name == name }
+  end
+end

data/lib/schema_reaper/static/scanner.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "prism"
+require "set"
+
+module SchemaReaper
+  module Static
+    # Walks the project and collects every identifier/symbol/string token that
+    # could name a database column. Over-collects on purpose: a false "used" is
+    # safe, a false "dead" is not.
+    class Scanner
+      RUBY_GLOB  = "**/*.rb"
+      WORD_RE    = /[a-z_][a-z0-9_]*/i
+
+      # Node classes whose #name (or #unescaped) is a bare identifier we treat
+      # as a possible column/table reference.
+      NAME_NODES = [
+        Prism::CallNode, Prism::DefNode, Prism::ConstantReadNode,
+        Prism::ConstantPathNode, Prism::ClassNode, Prism::ModuleNode,
+        Prism::LocalVariableReadNode, Prism::CallTargetNode
+      ].freeze
+
+      def initialize(config, root: Dir.pwd)
+        @config = config
+        @root = root
+      end
+
+      # @return [Set<String>] lowercased tokens seen anywhere in the codebase
+      def call
+        tokens = Set.new
+        ruby_files.each { |f| tokens.merge(ruby_tokens(f)) }
+        view_files.each { |f| tokens.merge(text_tokens(f)) }
+        tokens
+      end
+
+      private
+
+      def ruby_files
+        @config.scan_paths.flat_map do |p|
+          Dir.glob(File.join(@root, p, RUBY_GLOB))
+        end.uniq
+      end
+
+      def view_files
+        @config.view_globs.flat_map { |g| Dir.glob(File.join(@root, g)) }.uniq
+      end
+
+      def ruby_tokens(path)
+        src = File.read(path)
+        out = Set.new
+        result = Prism.parse(src)
+        collect_from_node(result.value, out)
+        # SQL string literals: pull bare words out of any string in the file.
+        src.scan(/["'`]([^"'`]{0,4000})["'`]/) { |(s)| out.merge(s.scan(WORD_RE).map(&:downcase)) }
+        out
+      rescue StandardError
+        text_tokens(path)
+      end
+
+      def collect_from_node(node, out)
+        return unless node.is_a?(Prism::Node)
+
+        out.merge(tokens_for(node))
+        node.compact_child_nodes.each { |c| collect_from_node(c, out) }
+      end
+
+      def tokens_for(node)
+        case node
+        when Prism::SymbolNode
+          [node.unescaped&.to_s&.downcase].compact
+        when Prism::StringNode
+          node.unescaped.to_s.scan(WORD_RE).map(&:downcase)
+        when *NAME_NODES
+          name = node.respond_to?(:name) ? node.name : nil
+          [name&.to_s&.downcase].compact
+        else
+          []
+        end
+      end
+
+      def text_tokens(path)
+        File.read(path).scan(WORD_RE).to_set(&:downcase)
+      rescue StandardError
+        Set.new
+      end
+    end
+  end
+end

data/lib/schema_reaper/tasks/schema_reaper.rake

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "schema_reaper"
+
+namespace :schema_reaper do
+  desc "Scan schema + code for dead weight (FORMAT=table|json|markdown|sarif)"
+  task scan: :environment do
+    findings = SchemaReaper::Runner.new.run
+    SchemaReaper.reporter(ENV.fetch("FORMAT", "table")).new(findings).render
+  end
+
+  desc "Record current findings to the baseline file"
+  task baseline: :environment do
+    findings = SchemaReaper::Runner.new.run
+    SchemaReaper::Baseline.new(SchemaReaper::Config.load.baseline_path).write(findings)
+  end
+
+  desc "Append a snapshot to the history log and print the trend"
+  task trend: :environment do
+    config = SchemaReaper::Config.load
+    findings = SchemaReaper::Runner.new(config: config).run
+    history = SchemaReaper::History.new(config.history_log)
+    history.record(findings)
+    pp history.trend
+  end
+end

data/lib/schema_reaper/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  VERSION = "1.0.0"
+end

data/lib/schema_reaper.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative "schema_reaper/version"
+require_relative "schema_reaper/config"
+require_relative "schema_reaper/finding"
+require_relative "schema_reaper/schema"
+require_relative "schema_reaper/runtime"
+require_relative "schema_reaper/gem_awareness"
+require_relative "schema_reaper/introspect/postgres"
+require_relative "schema_reaper/static/scanner"
+require_relative "schema_reaper/analyzers/base"
+require_relative "schema_reaper/analyzers/registry"
+require_relative "schema_reaper/analyzers/dead_column"
+require_relative "schema_reaper/analyzers/dead_table"
+require_relative "schema_reaper/analyzers/unused_index"
+require_relative "schema_reaper/analyzers/duplicate_index"
+require_relative "schema_reaper/analyzers/missing_fk_index"
+require_relative "schema_reaper/analyzers/always_null_column"
+require_relative "schema_reaper/analyzers/single_value_column"
+require_relative "schema_reaper/reporters/bytes"
+require_relative "schema_reaper/reporters/table"
+require_relative "schema_reaper/reporters/json"
+require_relative "schema_reaper/reporters/markdown"
+require_relative "schema_reaper/reporters/sarif"
+require_relative "schema_reaper/baseline"
+require_relative "schema_reaper/history"
+require_relative "schema_reaper/migration_generator"
+require_relative "schema_reaper/runner"
+require_relative "schema_reaper/railtie" if defined?(Rails::Railtie)
+
+# Finds columns, indexes and tables that a Rails/ActiveRecord app no longer
+# uses, then helps remove them safely. See {Runner} and the CLI.
+module SchemaReaper
+  class Error < StandardError; end
+
+  REPORTERS = {
+    "table" => Reporters::Table,
+    "json" => Reporters::Json,
+    "markdown" => Reporters::Markdown,
+    "sarif" => Reporters::Sarif
+  }.freeze
+
+  def self.reporter(name) = REPORTERS.fetch(name, Reporters::Table)
+end