activerecord-precompiled 0.1.0

data/lib/active_record_precompiled/schema_dump.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module ActiveRecordPrecompiled
+  # The build needs the production schema (columns, in order, with their
+  # types; primary keys), never rows. ActiveRecord's schema cache holds just
+  # that, so a dump made on production and committed lets the build run where
+  # the production database isn't, as in `docker build`: each SQLite database
+  # without a database file becomes an empty in-memory one that answers schema
+  # questions from its dump.
+  #
+  # The dumps are kept apart from ActiveRecord's own (db/schema_cache.yml),
+  # which the app would load at boot: at runtime the schema guard must compare
+  # against the real database.
+  module SchemaDump
+    MissingSchema = Class.new(StandardError)
+
+    PATH = "config/ar_precompile_schema_cache.yml"
+
+    # `mode`: "auto" (the default) uses a dump for each database whose file is
+    # missing; "dump" uses the dumps even if it isn't; "database" never does.
+    MODES = %w[auto dump database].freeze
+
+    class << self
+      # Writes the dump of each SQLite database; run it where the production
+      # database is, and commit the files.
+      def dump(root, out: nil)
+        sqlite_pools.uniq { it.db_config.name }.map do |pool|
+          path = path(root, pool.db_config)
+          ActiveRecord::Tasks::DatabaseTasks.dump_schema_cache(pool, path)
+          out&.puts "activerecord-precompiled: wrote the #{pool.db_config.name} schema to #{relative(root, path)}"
+          path
+        end
+      end
+
+      # Points the databases that need it at their dumps; the paths used.
+      def use(root, mode: nil, out: nil)
+        mode = (mode.nil? || mode.empty?) ? "auto" : mode
+        raise ArgumentError, "AR_PRECOMPILE_SCHEMA must be one of #{MODES.join(", ")}, not #{mode.inspect}" unless MODES.include?(mode)
+        return [] if mode == "database"
+
+        used = sqlite_pools.filter_map do |pool|
+          db_config = pool.db_config
+          missing = missing(root, pool)
+          next if mode == "auto" && !missing
+
+          path = path(root, db_config)
+          unless File.file?(path)
+            message = "no #{db_config.name} database at #{db_config.database} and no schema dump at #{relative(root, path)}"
+            hint = "run `RAILS_ENV=production bin/rails ar_precompile:schema:dump` where the production database is, and commit the file"
+            raise MissingSchema, "activerecord-precompiled: #{message}; #{hint}" if db_config.primary?
+
+            # Solid Cache/Queue/Cable and the like: chains on their models stay
+            # with ActiveRecord, and nothing creates an empty database file.
+            out&.puts "activerecord-precompiled: #{message}; queries on it are left to ActiveRecord (to compile them, #{hint})"
+            repoint(pool, nil)
+            remove(root, missing, out) if missing.is_a?(String)
+            next
+          end
+          out&.puts "activerecord-precompiled: #{db_config.name} schema from #{relative(root, path)}"
+          migrate(root, repoint(pool, path), path, out)
+          remove(root, missing, out) if missing.is_a?(String)
+          path
+        end
+        if used.any?
+          # The empty in-memory database has no schema_migrations to check the dump against.
+          ActiveRecord::ConnectionAdapters::SchemaReflection.use_schema_cache_dump = true
+          ActiveRecord::ConnectionAdapters::SchemaReflection.check_schema_cache_dump_version = false
+        end
+        used
+      end
+
+      # PATH for the primary database, config/ar_precompile_<name>_schema_cache.yml for others.
+      def path(root, db_config)
+        File.join(root.to_s, db_config.primary? ? PATH : "config/ar_precompile_#{db_config.name}_schema_cache.yml")
+      end
+
+      private
+
+      def sqlite_pools
+        ActiveRecord::Base.connection_handler.connection_pool_list(:all).select { it.db_config.adapter.to_s.start_with?("sqlite") }
+      end
+
+      # Whether the database is missing: true if its file is; the file if it
+      # has no tables, which is what connecting while the app loads, before
+      # the build, leaves behind (Rails 8's pragmas write a 4 KB header).
+      def missing(root, pool)
+        database = pool.db_config.database.to_s
+        return false if database.empty? || database == ":memory:" || database.start_with?("file:")
+
+        file = File.expand_path(database, root.to_s)
+        return true unless File.exist?(file)
+
+        pool.with_connection { it.data_sources.empty? } && file
+      end
+
+      # Out of the image: the app would start on it instead of failing.
+      def remove(root, file, out)
+        FileUtils.rm_f(["", "-wal", "-shm", "-journal"].map { file + it })
+        out&.puts "activerecord-precompiled: removed #{relative(root, file)}, a database with no tables (as connecting while the app loads creates)"
+      end
+
+      # The same connection (class, role, shard), on an empty in-memory
+      # database with the dump (if any) as its schema cache.
+      def repoint(pool, path)
+        config = pool.db_config
+        memory = ActiveRecord::DatabaseConfigurations::HashConfig.new(
+          config.env_name, config.name, config.configuration_hash.merge(database: ":memory:", schema_cache_path: path),
+        )
+        pool_config = pool.pool_config
+        ActiveRecord::Base.connection_handler.establish_connection(
+          memory, owner_name: pool_config.connection_descriptor, role: pool_config.role, shard: pool_config.shard,
+        )
+      end
+
+      # A dump older than the newest migration lacks its changes: rebuild the
+      # dumped tables, columns in order, and run the newer migrations on them,
+      # as production will. SQLite adds a column last and rebuilds a changed
+      # table in place, so the order comes out as production's. If a migration
+      # can't run on empty tables, the build uses the dump as it is, and the
+      # sites it changes fall back to ActiveRecord at runtime.
+      def migrate(root, pool, path, out)
+        dump = {}
+        ActiveRecord::ConnectionAdapters::SchemaCache._load_from(path).encode_with(dump)
+        dumped = dump["version"].to_i
+        pending = pool.migration_context.migrations.select { it.version > dumped }
+        return pool if pending.empty?
+
+        versions = pending.map(&:version).join(", ")
+        pool = repoint(pool, nil)
+        pool.with_connection do |connection|
+          dump["columns"].each { |table, columns| create_table(connection, table, columns, dump["primary_keys"][table]) }
+          dump["indexes"].each_value do |indexes|
+            indexes.each { connection.add_index(it.table, it.columns, name: it.name, unique: it.unique, where: it.where, order: it.orders) }
+          end
+          pool.schema_migration.create_table
+          pool.migration_context.migrations.each { pool.schema_migration.create_version(it.version.to_s) if it.version <= dumped }
+          verbose, ActiveRecord::Migration.verbose = ActiveRecord::Migration.verbose, false
+          begin
+            pool.migration_context.migrate
+          ensure
+            ActiveRecord::Migration.verbose = verbose
+          end
+        end
+        out&.puts "activerecord-precompiled: applied migrations #{versions} to #{relative(root, path)}; " \
+          "dump it again once they have run on production"
+        pool
+      rescue StandardError => e
+        e = e.cause || e # the Migrator's "this and all later migrations canceled"
+        message = "activerecord-precompiled: #{relative(root, path)} is at schema version #{dumped}, and migrations " \
+          "#{versions} didn't run on it (#{e.class}: #{e.message.lines.first&.strip}); building from it as it is, " \
+          "so the queries they change run on ActiveRecord: dump it again once they have run on production"
+        out ? out.puts(message) : ActiveRecordPrecompiled.warn(message)
+        repoint(pool, path)
+      end
+
+      # The table as the dump has it: columns in order, types, defaults, keys.
+      def create_table(connection, table, columns, primary_key)
+        keys = Array(primary_key)
+        definitions = columns.map do |column|
+          sql = +"#{connection.quote_column_name(column.name)} #{column.sql_type}"
+          sql << " PRIMARY KEY" if keys == [column.name]
+          sql << " AUTOINCREMENT" if keys == [column.name] && column.auto_increment?
+          sql << " NOT NULL" unless column.null
+          if column.default_function
+            sql << " DEFAULT (#{column.default_function})"
+          elsif !column.default.nil?
+            sql << " DEFAULT #{connection.quote(column.default)}"
+          end
+          sql << " COLLATE #{connection.quote_column_name(column.collation)}" if column.collation
+          sql
+        end
+        definitions << "PRIMARY KEY (#{keys.map { connection.quote_column_name(it) }.join(", ")})" if keys.size > 1
+        connection.execute("CREATE TABLE #{connection.quote_table_name(table)} (#{definitions.join(", ")})")
+      end
+
+      def relative(root, path) = path.delete_prefix("#{root}/")
+    end
+  end
+end

data/lib/active_record_precompiled/scope_analyzer.rb

@@ -0,0 +1,246 @@
+# frozen_string_literal: true
+
+require "prism"
+
+module ActiveRecordPrecompiled
+  # Decides whether a scope always builds the same SQL shape, so its SQL can
+  # be captured at build time. Static means: one chain of all/where/where.not/order/
+  # limit/offset and other static scopes, whose arguments are literals or the
+  # scope's own parameters. Enum scopes are static by construction.
+  # Anything else raises NotStatic with a reason and a hint.
+  class ScopeAnalyzer
+    NotStatic = Class.new(StandardError)
+    Analysis = Data.define(:arity, :param_columns)
+
+    def initialize
+      @trees = {}
+    end
+
+    # The belongs_to behind a non-column `where` key: `where(site: x)` binds
+    # x (or, for a record, its primary key) to `site_id`. Returns the
+    # reflection, or why the key can't be compiled.
+    def self.belongs_to_key(model, key)
+      reflection = model.reflect_on_association(key.to_sym)
+      return "`#{key}` is not a column of #{model.name}" unless reflection
+      return "`#{key}` is a #{reflection.macro} association; only belongs_to keys are compiled" unless reflection.belongs_to?
+      return "`#{key}` is a polymorphic association" if reflection.polymorphic?
+      return "`#{key}` has a composite key" if [reflection.foreign_key, Keys.join_primary_key(reflection)].any?(Array)
+
+      reflection
+    end
+
+    def analyze(model, name, depth = 0)
+      body = Scopes.body_for(model, name) or raise NotStatic, "`#{name}` is not a scope"
+      raise NotStatic, "#{model.name}'s table `#{model.table_name}` doesn't exist" if depth.zero? && !model.table_exists?
+      if body == :extension
+        raise NotStatic, "it has an extension block (`scope ... do ... end`); define those methods elsewhere"
+      end
+      if (redefined = redefinition(model, name))
+        raise NotStatic, "redefined by #{redefined}; remove that method or the scope"
+      end
+
+      analyze_body(model, body, depth)
+    end
+
+    # Scopes Rails generates: enum values, has_rich_text (with_rich_text_*),
+    # has_*_attached (with_attached_*). Their source interpolates names, but
+    # once defined they always build the same relation.
+    GENERATED_BY = %w[active_record/enum.rb action_text/attribute.rb active_storage/attached/model.rb].freeze
+
+    def generated_scope?(body)
+      body.respond_to?(:source_location) && body.arity.zero? &&
+        GENERATED_BY.any? { body.source_location&.first.to_s.end_with?(it) }
+    end
+
+    # "path:line" of a method that replaced the scope ActiveRecord generated.
+    def redefinition(model, name)
+      file, line = model.method(name).source_location
+      file.to_s.end_with?("active_record/scoping/named.rb") ? nil : "#{file}:#{line}"
+    end
+
+    # A lambda that isn't a named scope, such as an association's scope.
+    def analyze_proc(model, body) = analyze_body(model, body, 0)
+
+    private
+
+    def analyze_body(model, body, depth)
+      raise NotStatic, "it nests scopes too deeply" if depth > 5
+      return Analysis.new(0, {}) if generated_scope?(body)
+
+      file, line = body.source_location
+      raise NotStatic, "its source can't be found (defined with eval?)" unless file && File.file?(file)
+
+      lambdas = lambdas_at(file, line)
+      raise NotStatic, "its lambda can't be found at #{file}:#{line}" if lambdas.empty?
+      raise NotStatic, "#{file}:#{line} has more than one lambda; put the scope on its own line" if lambdas.size > 1
+
+      node = lambdas.first
+      params = parameters(node)
+      raise NotStatic, "it doesn't take the parameters its source shows" unless body.arity == params.size
+
+      statements = node.body&.body || []
+      raise NotStatic, "its body must be a single query chain" unless statements.size == 1
+
+      param_columns = {}
+      check_chain(statements.first, model, params, param_columns, depth)
+      Analysis.new(params.size, param_columns)
+    end
+
+    def lambdas_at(file, line)
+      tree = (@trees[file] ||= Prism.parse_file(file).value)
+      found = []
+      tree.breadth_first_search do |n|
+        if n.is_a?(Prism::LambdaNode) && n.location.start_line == line
+          found << n
+        elsif n.is_a?(Prism::CallNode) && %i[lambda proc].include?(n.name) && n.receiver.nil? &&
+            n.block.is_a?(Prism::BlockNode) && n.location.start_line == line
+          found << n.block
+        end
+        false
+      end
+      found
+    end
+
+    def parameters(node)
+      return [] if node.parameters.nil?
+      unless node.parameters.is_a?(Prism::BlockParametersNode)
+        raise NotStatic, "it uses `it` or numbered parameters; name them: ->(x) { ... }"
+      end
+
+      p = node.parameters.parameters
+      return [] if p.nil?
+      plain = p.optionals.empty? && p.rest.nil? && p.posts.empty? && p.keywords.empty? && p.keyword_rest.nil? && p.block.nil?
+      unless plain && p.requireds.all?(Prism::RequiredParameterNode)
+        raise NotStatic, "it takes optional, splat, keyword or block parameters; use plain required ones"
+      end
+
+      p.requireds.map(&:name)
+    end
+
+    def check_chain(expr, model, params, param_columns, depth)
+      calls = []
+      cur = expr
+      while cur.is_a?(Prism::CallNode)
+        calls.unshift(cur)
+        cur = cur.receiver
+      end
+      if cur || calls.empty?
+        raise NotStatic, "`#{code((cur || expr).slice)}` is not a query method; start the body with where/order/limit/offset or a scope"
+      end
+
+      calls.each_with_index do |call, i|
+        raise NotStatic, "`#{code(call.slice)}` has a block or `&.`" if call.block || call.safe_navigation?
+
+        # `where.not(hash)`: the bare `where` only opens the chain.
+        if call.name == :where && call.arguments.nil?
+          unless calls[i + 1]&.name == :not
+            raise NotStatic, "`#{code(call.slice)}.#{calls[i + 1]&.name}` is not supported; use where(...) or where.not(...)"
+          end
+
+          next
+        end
+        if call.name == :not && !(i.positive? && calls[i - 1].name == :where && calls[i - 1].arguments.nil?)
+          raise NotStatic, "`not` without `where`"
+        end
+
+        check_call(call, call.arguments&.arguments || [], model, params, param_columns, depth)
+      end
+    end
+
+    def check_call(call, nodes, model, params, param_columns, depth)
+      case call.name
+      when :all
+        raise NotStatic, "`all` takes no arguments" unless nodes.empty?
+      when :where, :not
+        unless nodes.size == 1 && (nodes.first.is_a?(Prism::KeywordHashNode) || nodes.first.is_a?(Prism::HashNode))
+          raise NotStatic, "`#{code(call.message_loc&.slice)}(#{code(call.arguments&.slice)})` uses an SQL string or other non-hash condition; " \
+            "use hash conditions like where(age: a)"
+        end
+        nodes.first.elements.each { check_condition(it, model, params, param_columns) }
+      when :order, :includes, :preload
+        nodes.each { literal!(it, "#{call.name} argument") }
+        raise NotStatic, "`#{call.name}` needs arguments" if nodes.empty?
+      when :limit, :offset
+        unless nodes.size == 1 && nodes.first.is_a?(Prism::IntegerNode)
+          raise NotStatic, "`#{call.name}` needs an integer literal"
+        end
+      else
+        unless Scopes.body_for(model, call.name)
+          raise NotStatic, "`#{call.name}` is not where/order/limit/offset/includes/preload/all or a scope"
+        end
+
+        begin
+          inner = analyze(model, call.name, depth + 1)
+        rescue NotStatic => e
+          raise NotStatic, "it uses `#{call.name}`, which can't be compiled: #{e.message}"
+        end
+        raise NotStatic, "`#{code(call.slice)}` passes the wrong number of arguments" unless inner.arity == nodes.size
+
+        nodes.each_with_index do |node, i|
+          if (index = param_index(node, params))
+            param_columns[index] ||= inner.param_columns[i]
+          else
+            literal!(node, "argument to #{call.name}")
+          end
+        end
+      end
+    end
+
+    def check_condition(assoc, model, params, param_columns)
+      unless assoc.is_a?(Prism::AssocNode) && (assoc.key.is_a?(Prism::SymbolNode) || assoc.key.is_a?(Prism::StringNode))
+        raise NotStatic, "`#{code(assoc.slice)}` is not a `column: value` condition"
+      end
+
+      column = assoc.key.unescaped
+      value = assoc.value
+      unless model.columns_hash.key?(column)
+        reflection = ScopeAnalyzer.belongs_to_key(model, column)
+        raise NotStatic, reflection if reflection.is_a?(String)
+        raise NotStatic, "`#{code(assoc.slice)}`: a range on an association" if value.is_a?(Prism::RangeNode)
+      end
+
+      if value.is_a?(Prism::RangeNode)
+        [value.left, value.right].compact.each { check_value(it, value, column, params, param_columns) }
+      else
+        check_value(value, value, column, params, param_columns)
+      end
+    end
+
+    # `whole` is the condition's value (the range, if `node` is one end of it).
+    def check_value(node, whole, column, params, param_columns)
+      if (index = param_index(node, params))
+        param_columns[index] = column
+      elsif !literal?(node)
+        fixed = whole.slice.sub(node.slice, "x")
+        raise NotStatic, "`#{code(node.slice)}` in `#{column}:` is computed when the scope runs; make it a parameter: " \
+          "->(x) { where(#{column}: #{code(fixed)}) } and pass the value from the caller"
+      end
+    end
+
+    # Source quoted in a reason: one line, and short.
+    def code(text)
+      text = text.to_s.gsub(/\s+/, " ").strip
+      text.size > 60 ? "#{text[0, 57]}..." : text
+    end
+
+    def param_index(node, params)
+      node.is_a?(Prism::LocalVariableReadNode) && node.depth.zero? ? params.index(node.name) : nil
+    end
+
+    def literal!(node, what)
+      raise NotStatic, "#{what} `#{code(node.slice)}` is not a literal" unless literal?(node)
+    end
+
+    def literal?(node)
+      case node
+      when Prism::SymbolNode, Prism::StringNode, Prism::IntegerNode, Prism::FloatNode,
+           Prism::NilNode, Prism::TrueNode, Prism::FalseNode then true
+      when Prism::ArrayNode then node.elements.all? { literal?(it) }
+      when Prism::RangeNode then [node.left, node.right].compact.all? { literal?(it) }
+      when Prism::HashNode, Prism::KeywordHashNode
+        node.elements.all? { it.is_a?(Prism::AssocNode) && literal?(it.key) && literal?(it.value) }
+      else false
+      end
+    end
+  end
+end

data/lib/active_record_precompiled/scope_check.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module ActiveRecordPrecompiled
+  # Lists every scope and whether the build can compile it, with the reason
+  # and a fix when it can't. Read-only; safe in any environment.
+  class ScopeCheck
+    Entry = Data.define(:model, :scope, :signature, :location, :ok, :reason)
+    Report = Data.define(:entries, :generated)
+
+    def initialize(models: nil, root: Dir.pwd)
+      @models = models
+      @root = root.to_s
+      @analyzer = ScopeAnalyzer.new
+    end
+
+    def run(out: $stdout)
+      entries = []
+      generated = 0
+      models.each do |model|
+        (model.instance_variable_get(:@__ar_precompiled_scopes) || {}).each do |name, body|
+          next generated += 1 if @analyzer.generated_scope?(body)
+
+          entries << entry(model, name, body)
+        end
+      end
+      print(entries, generated, out)
+      Report.new(entries, generated)
+    end
+
+    private
+
+    def models
+      @models || ActiveRecord::Base.descendants.reject(&:abstract_class?).select(&:name).sort_by(&:name)
+    end
+
+    def entry(model, name, body)
+      params = body.respond_to?(:parameters) ? body.parameters.map(&:last).compact : []
+      signature = "#{model.name}.#{name}#{"(#{params.join(", ")})" if params.any?}"
+      location = body.respond_to?(:source_location) && body.source_location ? relative(body.source_location.first(2).join(":")) : "?"
+      @analyzer.analyze(model, name)
+      Entry.new(model.name, name, signature, location, true, nil)
+    rescue ScopeAnalyzer::NotStatic => e
+      Entry.new(model.name, name, signature, location, false, relative(e.message))
+    rescue StandardError => e
+      Entry.new(model.name, name, signature, location, false, "#{e.class}: #{e.message.lines.first&.strip}")
+    end
+
+    def relative(text) = text.gsub("#{@root}/", "")
+
+    def print(entries, generated, out)
+      width = entries.map { it.signature.size }.max.to_i
+      entries.each do |e|
+        out.puts "#{e.ok ? "✅" : "❌"} #{e.signature.ljust(width)}  #{e.location}"
+        out.puts "   #{e.reason}" unless e.ok
+      end
+      out.puts "activerecord-precompiled: #{entries.count(&:ok)} of #{entries.size} scopes can be compiled (plus #{generated} generated by Rails: enum, rich text, attachments)"
+    end
+  end
+end

data/lib/active_record_precompiled/scopes.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module ActiveRecordPrecompiled
+  # Remembers every scope's body when the scope is defined, so the build can
+  # read its source and decide whether it is static. Installed when the gem
+  # loads, which Bundler does before the app's models.
+  module Scopes
+    module Recorder
+      def scope(name, body, &block)
+        super.tap { (@__ar_precompiled_scopes ||= {})[name.to_sym] = block ? :extension : body }
+      end
+    end
+
+    def self.install!
+      named = ActiveRecord::Scoping::Named::ClassMethods
+      named.prepend(Recorder) unless named <= Recorder
+    end
+
+    # The body given to `scope`, looked up through STI parents; :extension if
+    # the scope has an extension block; nil if `name` is not a scope.
+    def self.body_for(model, name)
+      model.ancestors.each do |klass|
+        break if klass == ActiveRecord::Base
+        next unless klass.is_a?(Class)
+
+        body = klass.instance_variable_get(:@__ar_precompiled_scopes)&.[](name.to_sym)
+        return body if body
+      end
+      nil
+    end
+  end
+end