nandi 0.8.0

data/lib/generators/nandi/migration/templates/migration.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class <%= class_name %> < Nandi::Migration
+  def up
+    # Migration instructions go here, eg:
+    # add_column :widgets, :size, :integer
+  end
+
+  def down
+    # Reverse migration instructions go here, eg:
+    # remove_column :widgets, :size
+  end
+end

data/lib/generators/nandi/not_null_check/USAGE

@@ -0,0 +1,19 @@
+Description:
+    Generates two new database migrations which will safely add a check that
+    a column is not null, and validate it separately.
+
+Example:
+    rails generate nandi:not_null_check foos bar
+
+    This will create:
+        db/safe_migrations/20190424123727_add_not_null_check_on_bar_to_foos.rb
+        db/safe_migrations/20190424123728_validate_not_null_check_on_bar_to_foos.rb
+
+Example:
+    rails generate nandi:not_null_check foos bar --validation-timeout 20000
+
+    This will create:
+        db/safe_migrations/20190424123727_add_not_null_check_on_bar_to_foos.rb
+        db/safe_migrations/20190424123728_validate_not_null_check_on_bar_to_foos.rb
+
+    The statement timeout in the second migration will be set to 20,000ms.

data/lib/generators/nandi/not_null_check/not_null_check_generator.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "nandi/formatting"
+
+module Nandi
+  class NotNullCheckGenerator < Rails::Generators::Base
+    include Nandi::Formatting
+
+    argument :table, type: :string
+    argument :column, type: :string
+    class_option :validation_timeout, type: :numeric, default: 15 * 60 * 1000
+
+    source_root File.expand_path("templates", __dir__)
+
+    attr_reader :add_not_null_check_name, :validate_not_null_check_name
+
+    def add_not_null_check
+      self.table = table.to_sym
+      self.column = column.to_sym
+
+      @add_not_null_check_name = "add_not_null_check_on_#{column}_to_#{table}"
+
+      template(
+        "add_not_null_check.rb",
+        "#{base_path}/#{timestamp}_#{add_not_null_check_name}.rb",
+      )
+    end
+
+    def validate_not_null_check
+      self.table = table.to_sym
+      self.column = column.to_sym
+
+      @validate_not_null_check_name = "validate_not_null_check_on_#{column}_to_#{table}"
+
+      template(
+        "validate_not_null_check.rb",
+        "#{base_path}/#{timestamp(1)}_#{validate_not_null_check_name}.rb",
+      )
+    end
+
+    private
+
+    def base_path
+      Nandi.config.migration_directory || "db/safe_migrations"
+    end
+
+    def timestamp(offset = 0)
+      (Time.now.utc + offset).strftime("%Y%m%d%H%M%S")
+    end
+
+    def name
+      "#{table}_check_#{column}_not_null"
+    end
+  end
+end

data/lib/generators/nandi/not_null_check/templates/add_not_null_check.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class <%= add_not_null_check_name.camelize %> < Nandi::Migration
+  def up
+    add_check_constraint <%= format_value(table) %>, <%= format_value(name) %>, "<%= column %> IS NOT NULL"
+  end
+
+  def down
+    drop_constraint <%= format_value(table) %>, <%= format_value(name) %>
+  end
+end

data/lib/generators/nandi/not_null_check/templates/validate_not_null_check.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class <%= validate_not_null_check_name.camelize %> < Nandi::Migration
+  def up
+    validate_constraint <%= format_value(@table) %>, <%= format_value(name) %>
+  end
+
+  def down; end
+end

data/lib/nandi.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "nandi/config"
+require "nandi/renderers"
+require "nandi/compiled_migration"
+require "active_support/core_ext/string/inflections"
+
+module Nandi
+  class Error < StandardError; end
+
+  class << self
+    def compile(files:)
+      compiled = files.
+        map { |f| CompiledMigration.build(f) }
+
+      yield compiled
+    end
+
+    def configure
+      yield config
+    end
+
+    def validator
+      Nandi::Validator
+    end
+
+    def config
+      @config ||= Config.new
+    end
+
+    def compiled_output_directory
+      Nandi.config.output_directory || "db/migrate"
+    end
+  end
+end

data/lib/nandi/compiled_migration.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "nandi/file_diff"
+
+module Nandi
+  class CompiledMigration
+    class InvalidMigrationError < StandardError; end
+
+    attr_reader :file_name, :source_file_path, :class_name
+
+    def self.build(source_file_path)
+      new(source_file_path)
+    end
+
+    def initialize(source_file_path)
+      @source_file_path = source_file_path
+      require source_file_path
+
+      @file_name, @class_name = /\d+_([a-z0-9_]+)\.rb\z/.match(source_file_path)[0..1]
+    end
+
+    def validate!
+      validation = migration.validate
+
+      unless validation.valid?
+        raise InvalidMigrationError, "Migration #{source_file_path} " \
+          "is not valid:\n#{validation.error_list}"
+      end
+
+      self
+    end
+
+    def body
+      @body ||= if migration_unchanged?
+                  File.read(output_path)
+                else
+                  validate!
+                  compiled_body
+                end
+    end
+
+    def output_path
+      "#{Nandi.compiled_output_directory}/#{file_name}"
+    end
+
+    def migration
+      @migration ||= class_name.camelize.constantize.new(Nandi.validator)
+    end
+
+    def compiled_digest
+      Digest::SHA256.hexdigest(body)
+    end
+
+    def source_digest
+      Digest::SHA256.hexdigest(File.read(source_file_path))
+    end
+
+    def migration_unchanged?
+      return false unless File.exist?(output_path)
+
+      source_migration_diff = Nandi::FileDiff.new(
+        file_path: source_file_path,
+        known_digest: Nandi::Lockfile.get(file_name).fetch(:source_digest),
+      )
+
+      compiled_migration_diff = Nandi::FileDiff.new(
+        file_path: output_path,
+        known_digest: Nandi::Lockfile.get(file_name).fetch(:compiled_digest),
+      )
+
+      source_migration_diff.unchanged? && compiled_migration_diff.unchanged?
+    end
+
+    private
+
+    def compiled_body
+      output = Nandi.config.renderer.generate(migration)
+
+      if Nandi.config.post_processor
+        Nandi.config.post_processor.call(output)
+      else
+        output
+      end
+    end
+  end
+end

data/lib/nandi/config.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "nandi/renderers"
+require "nandi/lockfile"
+
+module Nandi
+  class Config
+    # Most DDL changes take a very strict lock, but execute very quickly. For these
+    # the statement timeout should be very tight, so that if there's an unexpected
+    # delay the query queue does not back up.
+    DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT = 1_500
+    DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT = 5_000
+
+    DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT_LIMIT =
+      DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT
+    DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT_LIMIT =
+      DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT
+    DEFAULT_LOCKFILE_DIRECTORY = File.join(Dir.pwd, "db")
+    DEFAULT_CONCURRENT_TIMEOUT_LIMIT = 3_600_000
+    DEFAULT_COMPILE_FILES = "all"
+    # The rendering backend used to produce output. The only supported option
+    # at current is Nandi::Renderers::ActiveRecord, which produces ActiveRecord
+    # migrations.
+    # @return [Class]
+    attr_accessor :renderer
+
+    # The default lock timeout for migrations that take ACCESS EXCLUSIVE
+    # locks. Can be overridden by way of the `set_lock_timeout` class
+    # method in a given migration. Default: 1500ms.
+    # @return [Integer]
+    attr_accessor :access_exclusive_lock_timeout
+
+    # The default statement timeout for migrations that take ACCESS EXCLUSIVE
+    # locks. Can be overridden by way of the `set_statement_timeout` class
+    # method in a given migration. Default: 1500ms.
+    # @return [Integer]
+    attr_accessor :access_exclusive_statement_timeout
+
+    # The maximum lock timeout for migrations that take an ACCESS EXCLUSIVE
+    # lock and therefore block all reads and writes. Default: 5,000ms.
+    # @return [Integer]
+    attr_accessor :access_exclusive_statement_timeout_limit
+
+    # The maximum statement timeout for migrations that take an ACCESS
+    # EXCLUSIVE lock and therefore block all reads and writes. Default: 1500ms.
+    # @return [Integer]
+    attr_accessor :access_exclusive_lock_timeout_limit
+
+    # The minimum statement timeout for migrations that take place concurrently.
+    # Default: 3,600,000ms (ie, 3 hours).
+    # @return [Integer]
+    attr_accessor :concurrent_statement_timeout_limit
+
+    # The minimum lock timeout for migrations that take place concurrently.
+    # Default: 3,600,000ms (ie, 3 hours).
+    # @return [Integer]
+    attr_accessor :concurrent_lock_timeout_limit
+
+    # The directory for Nandi migrations. Default: `db/safe_migrations`
+    # @return [String]
+    attr_accessor :migration_directory
+
+    # The directory for output files. Default: `db/migrate`
+    # @return [String]
+    attr_accessor :output_directory
+
+    # The files to compile when the compile generator is run. Default: `all`
+    # May be one of the following:
+    # - 'all' compiles all files
+    # - 'git-diff' only files changed since last commit
+    # - a full or partial version timestamp, eg '20190101010101', '20190101'
+    # - a timestamp range , eg '>=20190101010101'
+    # @return [String]
+    attr_accessor :compile_files
+    #
+    # Directory where .nandilock.yml will be stored
+    # Defaults to project root
+    # @return [String]
+    attr_writer :lockfile_directory
+
+    # @api private
+    attr_reader :post_processor, :custom_methods
+
+    def initialize(renderer: Renderers::ActiveRecord)
+      @renderer = renderer
+      @access_exclusive_statement_timeout = DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT
+      @concurrent_lock_timeout_limit =
+        @concurrent_statement_timeout_limit =
+          DEFAULT_CONCURRENT_TIMEOUT_LIMIT
+      @custom_methods = {}
+      @access_exclusive_lock_timeout =
+        DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT
+      @access_exclusive_statement_timeout =
+        DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT
+      @access_exclusive_statement_timeout_limit =
+        DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT_LIMIT
+      @access_exclusive_lock_timeout_limit = DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT_LIMIT
+      @compile_files = DEFAULT_COMPILE_FILES
+      @lockfile_directory = DEFAULT_LOCKFILE_DIRECTORY
+    end
+
+    # Register a block to be called on output, for example a code formatter. Whatever is
+    # returned will be written to the output file.
+    # @yieldparam migration [string] The text of a compiled migration.
+    def post_process(&block)
+      @post_processor = block
+    end
+
+    # Register a custom DDL method.
+    # @param name [Symbol] The name of the method to create. This will be monkey-patched
+    #   into Nandi::Migration.
+    # @param klass [Class] The class to initialise with the arguments to the
+    #   method. It should define a `template` instance method which will return a
+    #   subclass of Cell::ViewModel from the Cells templating library and a
+    #   `procedure` method that returns the name of the method. It may optionally
+    #   define a `mixins` method, which will return an array of `Module`s to be
+    #   mixed into any migration that uses this method.
+    def register_method(name, klass)
+      custom_methods[name] = klass
+    end
+
+    def lockfile_directory
+      @lockfile_directory ||= Pathname.new(@lockfile_directory)
+    end
+  end
+end

data/lib/nandi/file_diff.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Nandi
+  class FileDiff
+    attr_reader :file_path, :known_digest
+
+    def initialize(file_path:, known_digest:)
+      @file_path = file_path
+      @known_digest = known_digest
+    end
+
+    def file_name
+      File.basename(file_path)
+    end
+
+    def body
+      File.read(file_path)
+    end
+
+    def digest
+      Digest::SHA256.hexdigest(body)
+    end
+
+    def unchanged?
+      !changed?
+    end
+
+    def changed?
+      known_digest != digest
+    end
+  end
+end

data/lib/nandi/file_matcher.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Nandi
+  class FileMatcher
+    TIMESTAMP_REGEX = /\A(?<operator>>|>=)?(?<timestamp>\d+)\z/.freeze
+
+    def self.call(*args, **kwargs)
+      new(*args, **kwargs).call
+    end
+
+    def initialize(files:, spec:)
+      @files = Set.new(files)
+      @spec = spec
+    end
+
+    def call
+      case spec
+      when "all"
+        Set.new(
+          files.reject { |f| ignored_filenames.include?(File.basename(f)) },
+        )
+      when "git-diff"
+        files.intersection(files_from_git_status)
+      when TIMESTAMP_REGEX
+        match_timestamp
+      end
+    end
+
+    private
+
+    def ignored_files
+      @ignored_files ||= if File.exist?(".nandiignore")
+                           File.read(".nandiignore").lines.map(&:strip)
+                         else
+                           []
+                         end
+    end
+
+    def ignored_filenames
+      ignored_files.map(&File.method(:basename))
+    end
+
+    def match_timestamp
+      match = TIMESTAMP_REGEX.match(spec)
+
+      case match[:operator]
+      when nil
+        files.select do |file|
+          file.start_with?(match[:timestamp])
+        end
+      when ">"
+        migrations_after((Integer(match[:timestamp]) + 1).to_s)
+      when ">="
+        migrations_after(match[:timestamp])
+      end.to_set
+    end
+
+    def migrations_after(minimum)
+      files.select { |file| file >= minimum }
+    end
+
+    def files_from_git_status
+      `
+        git status --porcelain --short --untracked-files=all |
+        cut -c4- |
+        xargs -n1 basename
+      `.lines.map(&:strip)
+    end
+
+    attr_reader :files, :spec
+  end
+end