nandi 1.0.1 → 2.0.0

data/lib/nandi/config.rb

@@ -2,68 +2,19 @@
 
 require "nandi/renderers"
 require "nandi/lockfile"
+require "nandi/multi_database"
 
 module Nandi
   class Config
-    # Most DDL changes take a very strict lock, but execute very quickly. For these
-    # the statement timeout should be very tight, so that if there's an unexpected
-    # delay the query queue does not back up.
-    DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT = 1_500
-    DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT = 5_000
-
-    DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT_LIMIT =
-      DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT
-    DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT_LIMIT =
-      DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT
-    DEFAULT_LOCKFILE_DIRECTORY = File.join(Dir.pwd, "db")
-    DEFAULT_CONCURRENT_TIMEOUT_LIMIT = 3_600_000
     DEFAULT_COMPILE_FILES = "all"
+    DEFAULT_LOCKFILE_DIRECTORY = File.join(Dir.pwd, "db")
+
     # The rendering backend used to produce output. The only supported option
     # at current is Nandi::Renderers::ActiveRecord, which produces ActiveRecord
     # migrations.
     # @return [Class]
     attr_accessor :renderer
 
-    # The default lock timeout for migrations that take ACCESS EXCLUSIVE
-    # locks. Can be overridden by way of the `set_lock_timeout` class
-    # method in a given migration. Default: 1500ms.
-    # @return [Integer]
-    attr_accessor :access_exclusive_lock_timeout
-
-    # The default statement timeout for migrations that take ACCESS EXCLUSIVE
-    # locks. Can be overridden by way of the `set_statement_timeout` class
-    # method in a given migration. Default: 1500ms.
-    # @return [Integer]
-    attr_accessor :access_exclusive_statement_timeout
-
-    # The maximum lock timeout for migrations that take an ACCESS EXCLUSIVE
-    # lock and therefore block all reads and writes. Default: 5,000ms.
-    # @return [Integer]
-    attr_accessor :access_exclusive_statement_timeout_limit
-
-    # The maximum statement timeout for migrations that take an ACCESS
-    # EXCLUSIVE lock and therefore block all reads and writes. Default: 1500ms.
-    # @return [Integer]
-    attr_accessor :access_exclusive_lock_timeout_limit
-
-    # The minimum statement timeout for migrations that take place concurrently.
-    # Default: 3,600,000ms (ie, 3 hours).
-    # @return [Integer]
-    attr_accessor :concurrent_statement_timeout_limit
-
-    # The minimum lock timeout for migrations that take place concurrently.
-    # Default: 3,600,000ms (ie, 3 hours).
-    # @return [Integer]
-    attr_accessor :concurrent_lock_timeout_limit
-
-    # The directory for Nandi migrations. Default: `db/safe_migrations`
-    # @return [String]
-    attr_accessor :migration_directory
-
-    # The directory for output files. Default: `db/migrate`
-    # @return [String]
-    attr_accessor :output_directory
-
     # The files to compile when the compile generator is run. Default: `all`
     # May be one of the following:
     # - 'all' compiles all files
@@ -72,7 +23,7 @@ module Nandi
     # - a timestamp range , eg '>=20190101010101'
     # @return [String]
     attr_accessor :compile_files
-    #
+
     # Directory where .nandilock.yml will be stored
     # Defaults to project root
     # @return [String]
@@ -83,18 +34,7 @@ module Nandi
 
     def initialize(renderer: Renderers::ActiveRecord)
       @renderer = renderer
-      @access_exclusive_statement_timeout = DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT
-      @concurrent_lock_timeout_limit =
-        @concurrent_statement_timeout_limit =
-          DEFAULT_CONCURRENT_TIMEOUT_LIMIT
       @custom_methods = {}
-      @access_exclusive_lock_timeout =
-        DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT
-      @access_exclusive_statement_timeout =
-        DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT
-      @access_exclusive_statement_timeout_limit =
-        DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT_LIMIT
-      @access_exclusive_lock_timeout_limit = DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT_LIMIT
       @compile_files = DEFAULT_COMPILE_FILES
       @lockfile_directory = DEFAULT_LOCKFILE_DIRECTORY
     end
@@ -119,6 +59,71 @@ module Nandi
       custom_methods[name] = klass
     end
 
+    # Register a database to compile migrations for.
+    def register_database(name, config = {})
+      multi_db_config.register(name, config)
+    end
+
+    def lockfile_path(database_name = nil)
+      File.join(lockfile_directory, databases.config(database_name).lockfile_name)
+    end
+
+    # Explicitly define getters for backwards compatibility when the database isnt specified.
+    # rubocop:disable Layout/LineLength
+    def migration_directory(database_name = nil) = config(database_name).migration_directory
+    def output_directory(database_name = nil) = config(database_name).output_directory
+    def access_exclusive_lock_timeout(database_name = nil) = config(database_name).access_exclusive_lock_timeout
+    def access_exclusive_lock_timeout_limit(database_name = nil) = config(database_name).access_exclusive_lock_timeout_limit
+    def access_exclusive_statement_timeout(database_name = nil) = config(database_name).access_exclusive_statement_timeout
+    def access_exclusive_statement_timeout_limit(database_name = nil) = config(database_name).access_exclusive_statement_timeout_limit
+    def concurrent_lock_timeout_limit(database_name = nil) = config(database_name).concurrent_lock_timeout_limit
+    def concurrent_statement_timeout_limit(database_name = nil) = config(database_name).concurrent_statement_timeout_limit
+    # rubocop:enable Layout/LineLength
+
+    # Delegate setter methods to the default database for backwards compatibility
+    delegate :migration_directory=,
+             :output_directory=,
+             :access_exclusive_lock_timeout=,
+             :access_exclusive_lock_timeout_limit=,
+             :access_exclusive_statement_timeout=,
+             :access_exclusive_statement_timeout_limit=,
+             :concurrent_lock_timeout_limit=,
+             :concurrent_statement_timeout_limit=,
+             to: :default
+
+    delegate :validate!, :default, :config, to: :databases
+
+    alias_method :database, :config
+
+    def databases
+      # If we've never registered any databases, use a single database with
+      # default values for backwards compatibility.
+      @multi_db_config.nil? ? single_db_config : @multi_db_config
+    end
+
+    def validate!
+      if @single_db_config && @multi_db_config
+        raise ArgumentError, "Cannot use multi and single database config. Config setters are now deprecated, " \
+                             "use only `register_database(name, config)` to configure Nandi."
+      end
+      databases.validate!
+    end
+
+    private
+
+    def single_db_config
+      # Pre-register the default database to ensure behavior is backwards compatible.
+      @single_db_config ||= begin
+        single_db_config = MultiDatabase.new
+        single_db_config.register(:primary, {})
+        single_db_config
+      end
+    end
+
+    def multi_db_config
+      @multi_db_config ||= MultiDatabase.new
+    end
+
     def lockfile_directory
       @lockfile_directory ||= Pathname.new(@lockfile_directory)
     end

data/lib/nandi/lockfile.rb

@@ -5,74 +5,94 @@ require "digest"
 
 module Nandi
   class Lockfile
+    attr_reader :db_name
+
     class << self
-      def file_present?
-        File.exist?(path)
+      # Registry pattern using class variables to maintain singleton instances
+      # per database. This ensures that lockfile operations for the same database
+      # always work with the same instance, maintaining consistency.
+      def for(db_name)
+        @instances ||= {}
+        # Handle nil by using :primary as default
+        key = db_name.nil? ? :primary : db_name.to_sym
+        @instances[key] ||= new(key)
       end
 
-      def create!
-        return if file_present?
-
-        File.write(path, {}.to_yaml)
+      def clear_instances!
+        @instances = {}
       end
 
-      def add(file_name:, source_digest:, compiled_digest:)
-        load!
+      private_class_method :new
+    end
 
-        lockfile[file_name] = {
-          source_digest: source_digest,
-          compiled_digest: compiled_digest,
-        }
-      end
+    def initialize(db_name = nil)
+      @db_name = db_name || Nandi.config.default.name
+    end
 
-      def get(file_name)
-        load!
+    def file_present?
+      File.exist?(path)
+    end
 
-        {
-          source_digest: lockfile.dig(file_name, :source_digest),
-          compiled_digest: lockfile.dig(file_name, :compiled_digest),
-        }
-      end
+    def create!
+      return if file_present?
 
-      def load!
-        return lockfile if lockfile
+      File.write(path, {}.to_yaml)
+    end
 
-        Nandi::Lockfile.create! unless Nandi::Lockfile.file_present?
+    def add(file_name:, source_digest:, compiled_digest:)
+      load!
 
-        @lockfile = YAML.safe_load_file(path).with_indifferent_access
-      end
+      @lockfile[file_name] = {
+        source_digest: source_digest,
+        compiled_digest: compiled_digest,
+      }
+    end
 
-      def persist!
-        # This is a somewhat ridiculous trick to avoid merge conflicts in git.
-        #
-        # Normally, new migrations are added to the bottom of the Nandi lockfile.
-        # This is relatively unfriendly to git's merge algorithm, and means that
-        # if someone merges a pull request with a completely unrelated migration,
-        # you'll have to rebase to get yours merged as the last line of the file
-        # will be seen as a conflict (both branches added content there).
-        #
-        # This is in contrast to something like Gemfile.lock, where changes tend
-        # to be distributed throughout the file. The idea behind sorting by
-        # SHA-256 hash is to distribute new Nandi lockfile entries evenly, but
-        # also stably through the file. It needs to be stable or we'd have even
-        # worse merge conflict problems (e.g. if we randomised the order on
-        # writing the file, the whole thing would conflict pretty much every time
-        # it was regenerated).
-        content = lockfile.to_h.deep_stringify_keys.sort_by do |k, _|
-          Digest::SHA256.hexdigest(k)
-        end.to_h.to_yaml
-
-        File.write(path, content)
-      end
+    def get(file_name)
+      load!
 
-      def path
-        File.join(
-          Nandi.config.lockfile_directory,
-          ".nandilock.yml",
-        )
-      end
+      {
+        source_digest: @lockfile.dig(file_name, :source_digest),
+        compiled_digest: @lockfile.dig(file_name, :compiled_digest),
+      }
+    end
+
+    def load!
+      return @lockfile if @lockfile
+
+      create! unless file_present?
+
+      @lockfile = YAML.safe_load_file(path).with_indifferent_access
+    end
+
+    def persist!
+      load!
+      # This is a somewhat ridiculous trick to avoid merge conflicts in git.
+      #
+      # Normally, new migrations are added to the bottom of the Nandi lockfile.
+      # This is relatively unfriendly to git's merge algorithm, and means that
+      # if someone merges a pull request with a completely unrelated migration,
+      # you'll have to rebase to get yours merged as the last line of the file
+      # will be seen as a conflict (both branches added content there).
+      #
+      # This is in contrast to something like Gemfile.lock, where changes tend
+      # to be distributed throughout the file. The idea behind sorting by
+      # SHA-256 hash is to distribute new Nandi lockfile entries evenly, but
+      # also stably through the file. It needs to be stable or we'd have even
+      # worse merge conflict problems (e.g. if we randomised the order on
+      # writing the file, the whole thing would conflict pretty much every time
+      # it was regenerated).
+      content = @lockfile.to_h.deep_stringify_keys.sort_by do |k, _|
+        Digest::SHA256.hexdigest(k)
+      end.to_h.to_yaml
+
+      File.write(path, content)
+    end
+
+    private
 
-      attr_accessor :lockfile
+    def path
+      Nandi.config.lockfile_path(@db_name)
     end
   end
 end

data/lib/nandi/migration_violations.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Nandi
+  class MigrationViolations
+    def initialize
+      @ungenerated_files = []
+      @handwritten_files = []
+      @out_of_date_files = []
+      @hand_edited_files = []
+    end
+
+    def add_ungenerated(missing_files, directory)
+      return if missing_files.empty?
+
+      full_paths = build_full_paths(missing_files, directory)
+      @ungenerated_files.concat(full_paths)
+    end
+
+    def add_handwritten(handwritten_files, directory)
+      return if handwritten_files.empty?
+
+      full_paths = build_full_paths(handwritten_files, directory)
+      @handwritten_files.concat(full_paths)
+    end
+
+    def add_out_of_date(changed_files, directory)
+      return if changed_files.empty?
+
+      full_paths = build_full_paths(changed_files, directory)
+      @out_of_date_files.concat(full_paths)
+    end
+
+    def add_hand_edited(altered_files, directory)
+      return if altered_files.empty?
+
+      full_paths = build_full_paths(altered_files, directory)
+      @hand_edited_files.concat(full_paths)
+    end
+
+    def any?
+      [@ungenerated_files, @handwritten_files, @out_of_date_files, @hand_edited_files].any?(&:any?)
+    end
+
+    def to_error_message
+      error_messages = []
+
+      error_messages << ungenerated_error if @ungenerated_files.any?
+      error_messages << handwritten_error if @handwritten_files.any?
+      error_messages << out_of_date_error if @out_of_date_files.any?
+      error_messages << hand_edited_error if @hand_edited_files.any?
+
+      error_messages.join("\n\n")
+    end
+
+    private
+
+    def build_full_paths(filenames, directory)
+      filenames.map { |filename| File.join(directory, filename) }
+    end
+
+    def format_file_list(files)
+      "  - #{files.sort.join("\n  - ")}"
+    end
+
+    def ungenerated_error
+      <<~ERROR.strip
+        The following migrations are pending generation:
+
+        #{format_file_list(@ungenerated_files)}
+
+        Please run `rails generate nandi:compile` to generate your migrations.
+      ERROR
+    end
+
+    def handwritten_error
+      <<~ERROR.strip
+        The following migrations have been written by hand, not generated:
+
+        #{format_file_list(@handwritten_files)}
+
+        Please use Nandi to generate your migrations. In exeptional cases, hand-written
+        ActiveRecord migrations can be added to the .nandiignore file. Doing so will
+        require additional review that will slow your PR down.
+      ERROR
+    end
+
+    def out_of_date_error
+      <<~ERROR.strip
+        The following migrations have changed but not been recompiled:
+
+        #{format_file_list(@out_of_date_files)}
+
+        Please recompile your migrations to make sure that the changes you expect are
+        applied.
+      ERROR
+    end
+
+    def hand_edited_error
+      <<~ERROR.strip
+        The following migrations have had their generated content altered:
+
+        #{format_file_list(@hand_edited_files)}
+
+        Please don't hand-edit generated migrations. If you want to write a regular
+        ActiveRecord::Migration, please do so and add it to .nandiignore. Note that
+        this will require additional review that will slow your PR down.
+      ERROR
+    end
+  end
+end

data/lib/nandi/multi_database.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+module Nandi
+  class MultiDatabase
+    class Database
+      # Most DDL changes take a very strict lock, but execute very quickly. For these
+      # the statement timeout should be very tight, so that if there's an unexpected
+      # delay the query queue does not back up.
+      DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT = 1_500
+      DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT = 5_000
+
+      DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT_LIMIT =
+        DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT
+      DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT_LIMIT =
+        DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT
+      DEFAULT_CONCURRENT_TIMEOUT_LIMIT = 3_600_000
+
+      DEFAULT_MIGRATION_DIRECTORY = "db/safe_migrations"
+      DEFAULT_OUTPUT_DIRECTORY = "db/migrate"
+
+      # The default lock timeout for migrations that take ACCESS EXCLUSIVE
+      # locks. Can be overridden by way of the `set_lock_timeout` class
+      # method in a given migration. Default: 1500ms.
+      # @return [Integer]
+      attr_accessor :access_exclusive_lock_timeout
+
+      # The default statement timeout for migrations that take ACCESS EXCLUSIVE
+      # locks. Can be overridden by way of the `set_statement_timeout` class
+      # method in a given migration. Default: 1500ms.
+      # @return [Integer]
+      attr_accessor :access_exclusive_statement_timeout
+
+      # The maximum lock timeout for migrations that take an ACCESS EXCLUSIVE
+      # lock and therefore block all reads and writes. Default: 5,000ms.
+      # @return [Integer]
+      attr_accessor :access_exclusive_statement_timeout_limit
+
+      # The maximum statement timeout for migrations that take an ACCESS
+      # EXCLUSIVE lock and therefore block all reads and writes. Default: 1500ms.
+      # @return [Integer]
+      attr_accessor :access_exclusive_lock_timeout_limit
+
+      # The minimum statement timeout for migrations that take place concurrently.
+      # Default: 3,600,000ms (ie, 3 hours).
+      # @return [Integer]
+      attr_accessor :concurrent_statement_timeout_limit
+
+      # The minimum lock timeout for migrations that take place concurrently.
+      # Default: 3,600,000ms (ie, 3 hours).
+      # @return [Integer]
+      attr_accessor :concurrent_lock_timeout_limit
+
+      # The directory for output files. Default: `db/migrate`
+      # @return [String]
+      attr_accessor :output_directory
+
+      attr_reader :name, :default
+
+      attr_accessor :migration_directory,
+                    :lockfile_name
+
+      def initialize(name:, config:)
+        @name = name
+        @default = @name == :primary || config[:default] == true
+
+        # Paths and files
+        @migration_directory = config[:migration_directory] || "db/#{path_prefix(name, default)}safe_migrations"
+        @output_directory = config[:output_directory] || "db/#{path_prefix(name, default)}migrate"
+        @lockfile_name = config[:lockfile_name] || ".#{path_prefix(name, default)}nandilock.yml"
+
+        timeout_limits(config)
+      end
+
+      private
+
+      def timeout_limits(config)
+        @access_exclusive_lock_timeout =
+          config[:access_exclusive_lock_timeout] || DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT
+        @access_exclusive_statement_timeout =
+          config[:access_exclusive_statement_timeout] || DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT
+        @access_exclusive_lock_timeout_limit =
+          config[:access_exclusive_lock_timeout_limit] || DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT_LIMIT
+        @access_exclusive_statement_timeout_limit =
+          config[:access_exclusive_statement_timeout_limit] || DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT_LIMIT
+        @concurrent_lock_timeout_limit =
+          config[:concurrent_lock_timeout_limit] || DEFAULT_CONCURRENT_TIMEOUT_LIMIT
+        @concurrent_statement_timeout_limit =
+          config[:concurrent_statement_timeout_limit] || DEFAULT_CONCURRENT_TIMEOUT_LIMIT
+      end
+
+      def path_prefix(name, default)
+        default ? "" : "#{name}_"
+      end
+    end
+
+    def initialize
+      @databases = {}
+    end
+
+    def config(name = nil)
+      # If name isnt specified, return config for the default database. This mimics behavior
+      # of the rails migration commands.
+      return default if name.nil?
+
+      name = name.to_sym
+      db_config = @databases[name]
+      raise ArgumentError, "Missing database configuration for #{name}" if db_config.nil?
+
+      db_config
+    end
+
+    def default
+      @databases.values.find(&:default)
+    end
+
+    def register(name, config)
+      name = name.to_sym
+      raise ArgumentError, "Database #{name} already registered" if @databases.key?(name)
+
+      @databases[name] = Database.new(name: name, config: config)
+    end
+
+    def names
+      @databases.keys
+    end
+
+    def validate!
+      enforce_default_db_for_multi_database!
+      enforce_names_for_multi_database!
+      validate_unique_migration_directories!
+      validate_unique_output_directories!
+    end
+
+    delegate :each, :map, to: :@databases
+
+    private
+
+    def enforce_default_db_for_multi_database!
+      # If there is a `primary` database, we take that as the default database
+      # following rails behavior. If not, we will validate that there is one specified
+      # default database using the `default: true` option.
+      if @databases.values.none?(&:default)
+        raise ArgumentError, "Missing default database. Specify a default database using the `default: true` option " \
+                             "or by registering `primary` as a database name."
+      end
+      if @databases.values.count(&:default) > 1
+        raise ArgumentError, "Multiple default databases specified: " \
+                             "#{@databases.values.select(&:default).map(&:name).join(', ')}"
+      end
+    end
+
+    def validate_unique_migration_directories!
+      paths = @databases.values.map(&:migration_directory).uniq.filter(&:present?)
+      if paths.length != @databases.values.length
+        raise ArgumentError,
+              "Unique migration directories must be specified for each database"
+      end
+    end
+
+    def validate_unique_output_directories!
+      paths = @databases.values.map(&:output_directory).uniq.filter(&:present?)
+      if paths.length != @databases.values.length
+        raise ArgumentError,
+              "Unique output directories must be specified for each database"
+      end
+    end
+
+    def enforce_names_for_multi_database!
+      # If we're in multi-db mode, enforce that all databases have a name
+      return if @databases.count <= 1
+
+      unknown_names = @databases.keys.select(&:nil?)
+      if unknown_names.any?
+        raise ArgumentError, "Databases must have a name in multi-db mode"
+      end
+    end
+  end
+end

data/lib/nandi/multi_db_generator.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Nandi
+  module MultiDbGenerator
+    def self.included(base)
+      base.class_option :database,
+                        default: nil,
+                        type: :string,
+                        desc: "Database to migrate in multi-database mode. " \
+                              "If not specified, uses specified default or primary database"
+    end
+
+    private
+
+    def db_name
+      options["database"]&.to_sym
+    end
+
+    def base_path
+      Nandi.config.migration_directory(db_name)
+    end
+  end
+end