nandi 0.9.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (76) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +477 -0
  3. data/Rakefile +8 -0
  4. data/exe/nandi-enforce +36 -0
  5. data/lib/generators/nandi/check_constraint/USAGE +19 -0
  6. data/lib/generators/nandi/check_constraint/check_constraint_generator.rb +52 -0
  7. data/lib/generators/nandi/check_constraint/templates/add_check_constraint.rb +15 -0
  8. data/lib/generators/nandi/check_constraint/templates/validate_check_constraint.rb +9 -0
  9. data/lib/generators/nandi/compile/USAGE +19 -0
  10. data/lib/generators/nandi/compile/compile_generator.rb +62 -0
  11. data/lib/generators/nandi/foreign_key/USAGE +47 -0
  12. data/lib/generators/nandi/foreign_key/foreign_key_generator.rb +91 -0
  13. data/lib/generators/nandi/foreign_key/templates/add_foreign_key.rb +13 -0
  14. data/lib/generators/nandi/foreign_key/templates/add_reference.rb +11 -0
  15. data/lib/generators/nandi/foreign_key/templates/validate_foreign_key.rb +9 -0
  16. data/lib/generators/nandi/migration/USAGE +9 -0
  17. data/lib/generators/nandi/migration/migration_generator.rb +24 -0
  18. data/lib/generators/nandi/migration/templates/migration.rb +13 -0
  19. data/lib/generators/nandi/not_null_check/USAGE +19 -0
  20. data/lib/generators/nandi/not_null_check/not_null_check_generator.rb +56 -0
  21. data/lib/generators/nandi/not_null_check/templates/add_not_null_check.rb +11 -0
  22. data/lib/generators/nandi/not_null_check/templates/validate_not_null_check.rb +9 -0
  23. data/lib/nandi.rb +35 -0
  24. data/lib/nandi/compiled_migration.rb +86 -0
  25. data/lib/nandi/config.rb +126 -0
  26. data/lib/nandi/file_diff.rb +32 -0
  27. data/lib/nandi/file_matcher.rb +72 -0
  28. data/lib/nandi/formatting.rb +79 -0
  29. data/lib/nandi/instructions.rb +19 -0
  30. data/lib/nandi/instructions/add_check_constraint.rb +23 -0
  31. data/lib/nandi/instructions/add_column.rb +24 -0
  32. data/lib/nandi/instructions/add_foreign_key.rb +40 -0
  33. data/lib/nandi/instructions/add_index.rb +50 -0
  34. data/lib/nandi/instructions/change_column_default.rb +23 -0
  35. data/lib/nandi/instructions/create_table.rb +83 -0
  36. data/lib/nandi/instructions/drop_constraint.rb +22 -0
  37. data/lib/nandi/instructions/drop_table.rb +21 -0
  38. data/lib/nandi/instructions/irreversible_migration.rb +15 -0
  39. data/lib/nandi/instructions/remove_column.rb +23 -0
  40. data/lib/nandi/instructions/remove_index.rb +41 -0
  41. data/lib/nandi/instructions/remove_not_null_constraint.rb +22 -0
  42. data/lib/nandi/instructions/validate_constraint.rb +22 -0
  43. data/lib/nandi/lockfile.rb +58 -0
  44. data/lib/nandi/migration.rb +363 -0
  45. data/lib/nandi/renderers.rb +7 -0
  46. data/lib/nandi/renderers/active_record.rb +13 -0
  47. data/lib/nandi/renderers/active_record/generate.rb +59 -0
  48. data/lib/nandi/renderers/active_record/instructions.rb +134 -0
  49. data/lib/nandi/safe_migration_enforcer.rb +143 -0
  50. data/lib/nandi/timeout_policies.rb +38 -0
  51. data/lib/nandi/timeout_policies/access_exclusive.rb +54 -0
  52. data/lib/nandi/timeout_policies/concurrent.rb +64 -0
  53. data/lib/nandi/validation.rb +10 -0
  54. data/lib/nandi/validation/add_column_validator.rb +43 -0
  55. data/lib/nandi/validation/each_validator.rb +32 -0
  56. data/lib/nandi/validation/failure_helpers.rb +35 -0
  57. data/lib/nandi/validation/remove_index_validator.rb +30 -0
  58. data/lib/nandi/validation/result.rb +30 -0
  59. data/lib/nandi/validation/timeout_validator.rb +37 -0
  60. data/lib/nandi/validator.rb +102 -0
  61. data/lib/nandi/version.rb +5 -0
  62. data/lib/templates/nandi/renderers/active_record/generate/show.rb.erb +27 -0
  63. data/lib/templates/nandi/renderers/active_record/instructions/add_check_constraint/show.rb.erb +7 -0
  64. data/lib/templates/nandi/renderers/active_record/instructions/add_column/show.rb.erb +6 -0
  65. data/lib/templates/nandi/renderers/active_record/instructions/add_foreign_key/show.rb.erb +5 -0
  66. data/lib/templates/nandi/renderers/active_record/instructions/add_index/show.rb.erb +5 -0
  67. data/lib/templates/nandi/renderers/active_record/instructions/change_column_default/show.rb.erb +1 -0
  68. data/lib/templates/nandi/renderers/active_record/instructions/create_table/show.rb.erb +8 -0
  69. data/lib/templates/nandi/renderers/active_record/instructions/drop_constraint/show.rb.erb +4 -0
  70. data/lib/templates/nandi/renderers/active_record/instructions/drop_table/show.rb.erb +1 -0
  71. data/lib/templates/nandi/renderers/active_record/instructions/irreversible_migration/show.rb.erb +1 -0
  72. data/lib/templates/nandi/renderers/active_record/instructions/remove_column/show.rb.erb +5 -0
  73. data/lib/templates/nandi/renderers/active_record/instructions/remove_index/show.rb.erb +4 -0
  74. data/lib/templates/nandi/renderers/active_record/instructions/remove_not_null_constraint/show.rb.erb +1 -0
  75. data/lib/templates/nandi/renderers/active_record/instructions/validate_constraint/show.rb.erb +3 -0
  76. metadata +320 -0
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ class <%= class_name %> < Nandi::Migration
4
+ def up
5
+ # Migration instructions go here, eg:
6
+ # add_column :widgets, :size, :integer
7
+ end
8
+
9
+ def down
10
+ # Reverse migration instructions go here, eg:
11
+ # remove_column :widgets, :size
12
+ end
13
+ end
@@ -0,0 +1,19 @@
1
+ Description:
2
+ Generates two new database migrations which will safely add a check that
3
+ a column is not null, and validate it separately.
4
+
5
+ Example:
6
+ rails generate nandi:not_null_check foos bar
7
+
8
+ This will create:
9
+ db/safe_migrations/20190424123727_add_not_null_check_on_bar_to_foos.rb
10
+ db/safe_migrations/20190424123728_validate_not_null_check_on_bar_to_foos.rb
11
+
12
+ Example:
13
+ rails generate nandi:not_null_check foos bar --validation-timeout 20000
14
+
15
+ This will create:
16
+ db/safe_migrations/20190424123727_add_not_null_check_on_bar_to_foos.rb
17
+ db/safe_migrations/20190424123728_validate_not_null_check_on_bar_to_foos.rb
18
+
19
+ The statement timeout in the second migration will be set to 20,000ms.
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails/generators"
4
+ require "nandi/formatting"
5
+
6
+ module Nandi
7
+ class NotNullCheckGenerator < Rails::Generators::Base
8
+ include Nandi::Formatting
9
+
10
+ argument :table, type: :string
11
+ argument :column, type: :string
12
+ class_option :validation_timeout, type: :numeric, default: 15 * 60 * 1000
13
+
14
+ source_root File.expand_path("templates", __dir__)
15
+
16
+ attr_reader :add_not_null_check_name, :validate_not_null_check_name
17
+
18
+ def add_not_null_check
19
+ self.table = table.to_sym
20
+ self.column = column.to_sym
21
+
22
+ @add_not_null_check_name = "add_not_null_check_on_#{column}_to_#{table}"
23
+
24
+ template(
25
+ "add_not_null_check.rb",
26
+ "#{base_path}/#{timestamp}_#{add_not_null_check_name}.rb",
27
+ )
28
+ end
29
+
30
+ def validate_not_null_check
31
+ self.table = table.to_sym
32
+ self.column = column.to_sym
33
+
34
+ @validate_not_null_check_name = "validate_not_null_check_on_#{column}_to_#{table}"
35
+
36
+ template(
37
+ "validate_not_null_check.rb",
38
+ "#{base_path}/#{timestamp(1)}_#{validate_not_null_check_name}.rb",
39
+ )
40
+ end
41
+
42
+ private
43
+
44
+ def base_path
45
+ Nandi.config.migration_directory || "db/safe_migrations"
46
+ end
47
+
48
+ def timestamp(offset = 0)
49
+ (Time.now.utc + offset).strftime("%Y%m%d%H%M%S")
50
+ end
51
+
52
+ def name
53
+ "#{table}_check_#{column}_not_null"
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ class <%= add_not_null_check_name.camelize %> < Nandi::Migration
4
+ def up
5
+ add_check_constraint <%= format_value(table) %>, <%= format_value(name) %>, "<%= column %> IS NOT NULL"
6
+ end
7
+
8
+ def down
9
+ drop_constraint <%= format_value(table) %>, <%= format_value(name) %>
10
+ end
11
+ end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ class <%= validate_not_null_check_name.camelize %> < Nandi::Migration
4
+ def up
5
+ validate_constraint <%= format_value(@table) %>, <%= format_value(name) %>
6
+ end
7
+
8
+ def down; end
9
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "nandi/config"
4
+ require "nandi/renderers"
5
+ require "nandi/compiled_migration"
6
+ require "active_support/core_ext/string/inflections"
7
+
8
+ module Nandi
9
+ class Error < StandardError; end
10
+
11
+ class << self
12
+ def compile(files:)
13
+ compiled = files.
14
+ map { |f| CompiledMigration.build(f) }
15
+
16
+ yield compiled
17
+ end
18
+
19
+ def configure
20
+ yield config
21
+ end
22
+
23
+ def validator
24
+ Nandi::Validator
25
+ end
26
+
27
+ def config
28
+ @config ||= Config.new
29
+ end
30
+
31
+ def compiled_output_directory
32
+ Nandi.config.output_directory || "db/migrate"
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "nandi/file_diff"
4
+
5
+ module Nandi
6
+ class CompiledMigration
7
+ class InvalidMigrationError < StandardError; end
8
+
9
+ attr_reader :file_name, :source_file_path, :class_name
10
+
11
+ def self.build(source_file_path)
12
+ new(source_file_path)
13
+ end
14
+
15
+ def initialize(source_file_path)
16
+ @source_file_path = source_file_path
17
+ require source_file_path
18
+
19
+ @file_name, @class_name = /\d+_([a-z0-9_]+)\.rb\z/.match(source_file_path)[0..1]
20
+ end
21
+
22
+ def validate!
23
+ validation = migration.validate
24
+
25
+ unless validation.valid?
26
+ raise InvalidMigrationError, "Migration #{source_file_path} " \
27
+ "is not valid:\n#{validation.error_list}"
28
+ end
29
+
30
+ self
31
+ end
32
+
33
+ def body
34
+ @body ||= if migration_unchanged?
35
+ File.read(output_path)
36
+ else
37
+ validate!
38
+ compiled_body
39
+ end
40
+ end
41
+
42
+ def output_path
43
+ "#{Nandi.compiled_output_directory}/#{file_name}"
44
+ end
45
+
46
+ def migration
47
+ @migration ||= class_name.camelize.constantize.new(Nandi.validator)
48
+ end
49
+
50
+ def compiled_digest
51
+ Digest::SHA256.hexdigest(body)
52
+ end
53
+
54
+ def source_digest
55
+ Digest::SHA256.hexdigest(File.read(source_file_path))
56
+ end
57
+
58
+ def migration_unchanged?
59
+ return false unless File.exist?(output_path)
60
+
61
+ source_migration_diff = Nandi::FileDiff.new(
62
+ file_path: source_file_path,
63
+ known_digest: Nandi::Lockfile.get(file_name).fetch(:source_digest),
64
+ )
65
+
66
+ compiled_migration_diff = Nandi::FileDiff.new(
67
+ file_path: output_path,
68
+ known_digest: Nandi::Lockfile.get(file_name).fetch(:compiled_digest),
69
+ )
70
+
71
+ source_migration_diff.unchanged? && compiled_migration_diff.unchanged?
72
+ end
73
+
74
+ private
75
+
76
+ def compiled_body
77
+ output = Nandi.config.renderer.generate(migration)
78
+
79
+ if Nandi.config.post_processor
80
+ Nandi.config.post_processor.call(output)
81
+ else
82
+ output
83
+ end
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,126 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "nandi/renderers"
4
+ require "nandi/lockfile"
5
+
6
+ module Nandi
7
+ class Config
8
+ # Most DDL changes take a very strict lock, but execute very quickly. For these
9
+ # the statement timeout should be very tight, so that if there's an unexpected
10
+ # delay the query queue does not back up.
11
+ DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT = 1_500
12
+ DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT = 5_000
13
+
14
+ DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT_LIMIT =
15
+ DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT
16
+ DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT_LIMIT =
17
+ DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT
18
+ DEFAULT_LOCKFILE_DIRECTORY = File.join(Dir.pwd, "db")
19
+ DEFAULT_CONCURRENT_TIMEOUT_LIMIT = 3_600_000
20
+ DEFAULT_COMPILE_FILES = "all"
21
+ # The rendering backend used to produce output. The only supported option
22
+ # at current is Nandi::Renderers::ActiveRecord, which produces ActiveRecord
23
+ # migrations.
24
+ # @return [Class]
25
+ attr_accessor :renderer
26
+
27
+ # The default lock timeout for migrations that take ACCESS EXCLUSIVE
28
+ # locks. Can be overridden by way of the `set_lock_timeout` class
29
+ # method in a given migration. Default: 1500ms.
30
+ # @return [Integer]
31
+ attr_accessor :access_exclusive_lock_timeout
32
+
33
+ # The default statement timeout for migrations that take ACCESS EXCLUSIVE
34
+ # locks. Can be overridden by way of the `set_statement_timeout` class
35
+ # method in a given migration. Default: 1500ms.
36
+ # @return [Integer]
37
+ attr_accessor :access_exclusive_statement_timeout
38
+
39
+ # The maximum lock timeout for migrations that take an ACCESS EXCLUSIVE
40
+ # lock and therefore block all reads and writes. Default: 5,000ms.
41
+ # @return [Integer]
42
+ attr_accessor :access_exclusive_statement_timeout_limit
43
+
44
+ # The maximum statement timeout for migrations that take an ACCESS
45
+ # EXCLUSIVE lock and therefore block all reads and writes. Default: 1500ms.
46
+ # @return [Integer]
47
+ attr_accessor :access_exclusive_lock_timeout_limit
48
+
49
+ # The minimum statement timeout for migrations that take place concurrently.
50
+ # Default: 3,600,000ms (ie, 3 hours).
51
+ # @return [Integer]
52
+ attr_accessor :concurrent_statement_timeout_limit
53
+
54
+ # The minimum lock timeout for migrations that take place concurrently.
55
+ # Default: 3,600,000ms (ie, 3 hours).
56
+ # @return [Integer]
57
+ attr_accessor :concurrent_lock_timeout_limit
58
+
59
+ # The directory for Nandi migrations. Default: `db/safe_migrations`
60
+ # @return [String]
61
+ attr_accessor :migration_directory
62
+
63
+ # The directory for output files. Default: `db/migrate`
64
+ # @return [String]
65
+ attr_accessor :output_directory
66
+
67
+ # The files to compile when the compile generator is run. Default: `all`
68
+ # May be one of the following:
69
+ # - 'all' compiles all files
70
+ # - 'git-diff' only files changed since last commit
71
+ # - a full or partial version timestamp, eg '20190101010101', '20190101'
72
+ # - a timestamp range , eg '>=20190101010101'
73
+ # @return [String]
74
+ attr_accessor :compile_files
75
+ #
76
+ # Directory where .nandilock.yml will be stored
77
+ # Defaults to project root
78
+ # @return [String]
79
+ attr_writer :lockfile_directory
80
+
81
+ # @api private
82
+ attr_reader :post_processor, :custom_methods
83
+
84
+ def initialize(renderer: Renderers::ActiveRecord)
85
+ @renderer = renderer
86
+ @access_exclusive_statement_timeout = DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT
87
+ @concurrent_lock_timeout_limit =
88
+ @concurrent_statement_timeout_limit =
89
+ DEFAULT_CONCURRENT_TIMEOUT_LIMIT
90
+ @custom_methods = {}
91
+ @access_exclusive_lock_timeout =
92
+ DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT
93
+ @access_exclusive_statement_timeout =
94
+ DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT
95
+ @access_exclusive_statement_timeout_limit =
96
+ DEFAULT_ACCESS_EXCLUSIVE_STATEMENT_TIMEOUT_LIMIT
97
+ @access_exclusive_lock_timeout_limit = DEFAULT_ACCESS_EXCLUSIVE_LOCK_TIMEOUT_LIMIT
98
+ @compile_files = DEFAULT_COMPILE_FILES
99
+ @lockfile_directory = DEFAULT_LOCKFILE_DIRECTORY
100
+ end
101
+
102
+ # Register a block to be called on output, for example a code formatter. Whatever is
103
+ # returned will be written to the output file.
104
+ # @yieldparam migration [string] The text of a compiled migration.
105
+ def post_process(&block)
106
+ @post_processor = block
107
+ end
108
+
109
+ # Register a custom DDL method.
110
+ # @param name [Symbol] The name of the method to create. This will be monkey-patched
111
+ # into Nandi::Migration.
112
+ # @param klass [Class] The class to initialise with the arguments to the
113
+ # method. It should define a `template` instance method which will return a
114
+ # subclass of Cell::ViewModel from the Cells templating library and a
115
+ # `procedure` method that returns the name of the method. It may optionally
116
+ # define a `mixins` method, which will return an array of `Module`s to be
117
+ # mixed into any migration that uses this method.
118
+ def register_method(name, klass)
119
+ custom_methods[name] = klass
120
+ end
121
+
122
+ def lockfile_directory
123
+ @lockfile_directory ||= Pathname.new(@lockfile_directory)
124
+ end
125
+ end
126
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nandi
4
+ class FileDiff
5
+ attr_reader :file_path, :known_digest
6
+
7
+ def initialize(file_path:, known_digest:)
8
+ @file_path = file_path
9
+ @known_digest = known_digest
10
+ end
11
+
12
+ def file_name
13
+ File.basename(file_path)
14
+ end
15
+
16
+ def body
17
+ File.read(file_path)
18
+ end
19
+
20
+ def digest
21
+ Digest::SHA256.hexdigest(body)
22
+ end
23
+
24
+ def unchanged?
25
+ !changed?
26
+ end
27
+
28
+ def changed?
29
+ known_digest != digest
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nandi
4
+ class FileMatcher
5
+ TIMESTAMP_REGEX = /\A(?<operator>>|>=)?(?<timestamp>\d+)\z/.freeze
6
+
7
+ def self.call(*args, **kwargs)
8
+ new(*args, **kwargs).call
9
+ end
10
+
11
+ def initialize(files:, spec:)
12
+ @files = Set.new(files)
13
+ @spec = spec
14
+ end
15
+
16
+ def call
17
+ case spec
18
+ when "all"
19
+ Set.new(
20
+ files.reject { |f| ignored_filenames.include?(File.basename(f)) },
21
+ )
22
+ when "git-diff"
23
+ files.intersection(files_from_git_status)
24
+ when TIMESTAMP_REGEX
25
+ match_timestamp
26
+ end
27
+ end
28
+
29
+ private
30
+
31
+ def ignored_files
32
+ @ignored_files ||= if File.exist?(".nandiignore")
33
+ File.read(".nandiignore").lines.map(&:strip)
34
+ else
35
+ []
36
+ end
37
+ end
38
+
39
+ def ignored_filenames
40
+ ignored_files.map(&File.method(:basename))
41
+ end
42
+
43
+ def match_timestamp
44
+ match = TIMESTAMP_REGEX.match(spec)
45
+
46
+ case match[:operator]
47
+ when nil
48
+ files.select do |file|
49
+ file.start_with?(match[:timestamp])
50
+ end
51
+ when ">"
52
+ migrations_after((Integer(match[:timestamp]) + 1).to_s)
53
+ when ">="
54
+ migrations_after(match[:timestamp])
55
+ end.to_set
56
+ end
57
+
58
+ def migrations_after(minimum)
59
+ files.select { |file| file >= minimum }
60
+ end
61
+
62
+ def files_from_git_status
63
+ `
64
+ git status --porcelain --short --untracked-files=all |
65
+ cut -c4- |
66
+ xargs -n1 basename
67
+ `.lines.map(&:strip)
68
+ end
69
+
70
+ attr_reader :files, :spec
71
+ end
72
+ end