pg_trunk 0.1.0

data/lib/pg_trunk/operations/indexes/add_index.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module PGTrunk::Operations::Indexes
+  # @private
+  #
+  # PGTrunk excludes indexes from table definitions provided by Rails.
+  # That's why we have to fetch and dump indexes separately.
+  #
+  # We fetch indexes from the database by their names and oids,
+  # and then rely on the original method +ActiveRecord::SchemaDumper#add_index+
+  #
+  # We doesn't overload the method `create_table`, but
+  # keep the original implementation unchanged. That's why
+  # neither `to_sql`, `invert` or `generates_object` are necessary.
+  #
+  class AddIndex < PGTrunk::Operation
+    attribute :table, :pg_trunk_qualified_name
+
+    validates :oid, :table, presence: true
+
+    # Indexes are ordered by table and name
+    def <=>(other)
+      return unless other.is_a?(self.class)
+
+      result = table <=> other.table
+      result&.zero? ? super : result
+    end
+
+    # SQL to fetch table names and oids from the database.
+    # We only extract (oid, table, name) for indexes that
+    # are not used as primary key constraints.
+    #
+    # Primary keys are added inside tables because
+    # they cannot depend on anything else.
+    from_sql do
+      <<~SQL
+        SELECT
+          c.oid,
+          (c.relnamespace::regnamespace || '.' || c.relname) AS name,
+          (t.relnamespace::regnamespace || '.' || t.relname) AS "table"
+        FROM pg_class c
+          -- ensure the table was created by a migration
+          JOIN pg_trunk p ON p.oid = c.oid
+          JOIN pg_index i ON i.indexrelid = c.oid
+          JOIN pg_class t ON t.oid = i.indrelid
+        -- ignore primary keys
+        WHERE NOT i.indisprimary
+      SQL
+    end
+
+    # Instead of defining +ruby_snippet+, we overload
+    # the +to_ruby+ to rely on the original implementation.
+    #
+    # We overloaded the +ActiveRecord::SchemaDumper+
+    # method +indexes_in_create+ so that it does nothing
+    # to exclude indexes from a table definition.
+    #
+    # @see +PGTrunk::SchemaDumper+ module (in `core/railtie`).
+    def to_ruby
+      indexes = PGTrunk.database.send(:indexes, table.lean)
+      index = indexes.find { |i| i.name == name.lean }
+      return unless index
+
+      line = PGTrunk.dumper.send(:index_parts, index).join(", ")
+      "add_index #{table.lean.inspect}, #{line}"
+    end
+  end
+end

data/lib/pg_trunk/operations/indexes.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# nodoc
+module PGTrunk::Operations
+  # @private
+  # Namespace for operations with indexes
+  module Indexes
+    require_relative "indexes/add_index"
+  end
+end

data/lib/pg_trunk/operations/materialized_views/base.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: false
+
+module PGTrunk::Operations::MaterializedViews
+  # @abstract
+  # @private
+  # Base class for operations with views
+  class Base < PGTrunk::Operation
+    # All attributes that can be used by view-related commands
+    attribute :algorithm, :pg_trunk_symbol
+    attribute :cluster_on, :string
+    attribute :columns, :pg_trunk_array_of_hashes, default: []
+    attribute :sql_definition, :pg_trunk_multiline_text
+    attribute :tablespace, :string
+    attribute :version, :integer, aliases: :revert_to_version
+    attribute :with_data, :boolean
+
+    def column(name, **opts)
+      columns << Column.new(name: name, **opts.except(:new_name))
+    end
+
+    # Load missed `sql_definition` from the external file
+    after_initialize { self.sql_definition ||= read_snippet_from(:materialized_views) }
+    after_initialize { columns.map! { |c| Column.build(c) } }
+
+    # Ensure correctness of present values
+    validates :algorithm, inclusion: %i[concurrently], allow_nil: true
+    validates :tablespace, exclusion: { in: [UNDEFINED] }, allow_nil: true
+    validates :columns, "PGTrunk/all_items_valid": true, allow_nil: true
+
+    # Use comparison by name from pg_trunk operations base class (default)
+    # Support name as the only positional argument (default)
+
+    ruby_snippet do |s|
+      s.ruby_param(name.lean) if name.present?
+      s.ruby_param(version: version) if version.present?
+      s.ruby_param(to: new_name.lean) if new_name.present?
+      s.ruby_param(if_exists: true) if if_exists
+      s.ruby_param(if_not_exists: true) if if_not_exists
+      s.ruby_param(force: :cascade) if force == :cascade
+
+      s.ruby_line(:sql_definition, sql_definition) if version.blank?
+      s.ruby_line(:tablespace, tablespace) if tablespace.present?
+      s.ruby_line(:cluster_on, cluster_on) if cluster_on.present?
+      columns.reject(&:new_name).each do |c|
+        s.ruby_line(:column, c.name, **c.changes)
+      end
+      columns.select(&:new_name).each do |c|
+        s.ruby_line(:rename_column, c.name, to: c.new_name)
+      end
+      s.ruby_line(:with_data, false) if with_data == false
+      s.ruby_line(:comment, comment, from: from_comment) if comment
+    end
+
+    private
+
+    # A special constant to distinct cluster resetting from nil
+    RESET = Object.new.freeze
+
+    def validate_naming!(name: nil, **)
+      errors.add :columns, "has undefined names" if name.blank?
+    end
+
+    def validate_definition!(name: nil, **opts)
+      return if opts.none? { |_, value| value == UNDEFINED }
+
+      errors.add :base, "Definition of column #{name} can't be reverted"
+    end
+
+    def validate_statistics!(name: nil, **opts)
+      opts.values_at(*STATISTICS).each do |value|
+        next if value.nil? || value == UNDEFINED
+        next if value.is_a?(Numeric) && value >= 0
+
+        errors.add :base, "Column #{name} has invalid statistics #{value}"
+        break
+      end
+    end
+  end
+end

data/lib/pg_trunk/operations/materialized_views/change_materialized_view.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: false
+
+# @!method ActiveRecord::Migration#change_materialized_view(name, **options, &block)
+# Modify a materialized view
+#
+# @param [#to_s] name (nil) The qualified name of the view
+# @option [Boolean] :if_exists (false) Suppress the error when the view is absent
+# @yield [Proc] the block with the view's definition
+# @yieldparam The receiver of methods specifying the view
+#
+# The operation enables to alter a view without recreating
+# its from scratch. You can rename columns, change their
+# storage settings (how the column is TOAST-ed), or
+# customize their statistics.
+#
+#   change_materialized_view "admin_users" do |v|
+#     v.rename_column "name", to: "full_name"
+#     v.column "name", storage: "extended", from_storage: "expanded"
+#     v.column "admin", n_distinct: 2
+#     v.column "role", statistics: 100
+#   end
+#
+# Notice that renaming will be done AFTER all changes even
+# though the order of declarations can be different.
+#
+# As in the snippet above, to make the change invertible,
+# you have to define a previous storage via `from_storage` option.
+# The inversion would always reset statistics (set it to 0).
+#
+# In addition to changing columns, the operation enables
+# to set a default clustering by given index:
+#
+#   change_materialized_view "admin_users" do |v|
+#     v.cluster_on "admin_users_by_names_idx"
+#   end
+#
+# The clustering is invertible, but its inversion does nothing,
+# keeping the clustering unchanged.
+#
+# The comment can also be changed:
+#
+#   change_materialized_view "admin_users" do |v|
+#     v.comment "Admin users", from: "Admin users only"
+#   end
+#
+# Notice, that without `from` option the operation is still
+# invertible, but its inversion would delete the comment.
+# It can also be reset to the blank string explicitly:
+#
+#   change_materialized_view "admin_users" do |v|
+#     v.comment "", from: "Admin users only"
+#   end
+#
+# With the `if_exists: true` option, the operation won't fail
+# even when the view wasn't existed. At the same time,
+# this option makes a migration irreversible due to uncertainty
+# of the previous state of the database.
+
+module PGTrunk::Operations::MaterializedViews
+  # @private
+  class ChangeMaterializedView < Base
+    # A method to be called in a block
+    def rename_column(name, to:)
+      columns << Column.new(name: name, new_name: to)
+    end
+
+    # Operation-specific validations
+    validate { errors.add :base, "Changes can't be blank" if changes.blank? }
+    validates :algorithm, :force, :if_not_exists, :new_name, :sql_definition,
+              :tablespace, :version, :with_data, absence: true
+
+    def to_sql(_version)
+      [
+        *change_columns,
+        *rename_columns,
+        *cluster_view,
+        *update_comment,
+      ].join(" ")
+    end
+
+    def invert
+      irreversible!("if_exists: true") if if_exists
+      undefined = inversion.select { |_, v| v.nil? }.keys.join(", ").presence
+      raise IrreversibleMigration.new(self, nil, <<~MSG.squish) if undefined
+        Undefined values to revert #{undefined}.
+      MSG
+
+      self.class.new(name: name, **inversion) if inversion.any?
+    end
+
+    private
+
+    def changes
+      @changes ||= {
+        columns: columns.presence,
+        cluster_on: cluster_on,
+        comment: comment,
+      }.compact
+    end
+
+    def inversion
+      @inversion ||= {
+        columns: columns.map(&:invert).presence,
+        comment: from_comment,
+      }.slice(*changes.keys)
+    end
+
+    def alter_view
+      @alter_view ||= begin
+        sql = "ALTER MATERIALIZED VIEW"
+        sql << " IF EXISTS" if if_exists
+        sql << " #{name.to_sql}"
+      end
+    end
+
+    def change_columns
+      changes = columns.reject(&:new_name).map(&:to_sql).join(", ")
+      "#{alter_view} #{changes};" if changes.present?
+    end
+
+    def rename_columns
+      changes = columns.select(&:new_name).map(&:to_sql).join(", ")
+      "#{alter_view} #{changes};" if changes.present?
+    end
+
+    def cluster_view
+      "#{alter_view} CLUSTER ON #{cluster_on.inspect};" if cluster_on.present?
+    end
+
+    def update_comment
+      return if comment.nil?
+
+      <<~SQL
+        COMMENT ON MATERIALIZED VIEW #{name.to_sql}
+        IS $comment$#{comment}$comment$;
+      SQL
+    end
+  end
+end

data/lib/pg_trunk/operations/materialized_views/column.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module PGTrunk::Operations::MaterializedViews
+  # @private
+  # Definition for the column change
+  class Column
+    include ActiveModel::Model
+    include ActiveModel::Attributes
+    include ActiveModel::Validations
+
+    def self.build(data)
+      data.is_a?(self) ? data : new(**data)
+    end
+
+    attribute :name, :string
+    attribute :new_name, :string
+    attribute :storage, :pg_trunk_symbol
+    attribute :from_storage, :pg_trunk_symbol
+    attribute :statistics, :integer
+    attribute :n_distinct, :float
+
+    # Hashify definitions
+
+    def to_h
+      @to_h ||=
+        attributes
+        .symbolize_keys
+        .transform_values(&:presence)
+        .compact
+    end
+
+    def opts
+      to_h.except(:name)
+    end
+
+    def changes
+      opts.except(:new_name)
+    end
+
+    def invert
+      return { name: new_name, new_name: name } if new_name.present?
+
+      {
+        name: name,
+        storage: (from_storage || :UNDEFINED if storage.present?),
+        statistics: (0 if statistics.present?),
+        n_distinct: (0 if n_distinct.present?),
+      }.compact
+    end
+
+    # Ensure if the definition was built properly
+
+    validates :name, presence: true
+    validate { errors.add(:base, :blank) if opts.none? }
+    validates :statistics,
+              numericality: { greater_than_or_equal_to: 0 },
+              allow_nil: true
+    validates :n_distinct,
+              numericality: { greater_than_or_equal_to: -1 },
+              allow_nil: true
+    validates :storage, :from_storage,
+              inclusion: { in: %i[plain extended external main] },
+              allow_nil: true
+    validate do
+      next unless n_distinct&.positive?
+      next if n_distinct.to_i == n_distinct
+
+      errors.add :n_distinct, "with positive value must be integer"
+    end
+
+    def error_messages
+      validate
+      errors&.messages&.flat_map do |k, v|
+        v.map do |msg|
+          "Column #{name.inspect}: #{k == :base ? msg : "#{k} #{msg}"}"
+        end
+      end
+    end
+
+    # Build SQL snippets for the column definition
+    # @return [Array<String>]
+    def to_sql(_version = "10")
+      return ["RENAME COLUMN #{name.inspect} TO #{new_name.inspect}"] if new_name
+
+      alter = "ALTER COLUMN #{name.inspect}"
+      [
+        *("#{alter} SET STATISTICS #{statistics}" if statistics),
+        *("#{alter} SET (n_distinct = #{n_distinct})" if n_distinct),
+        *("#{alter} RESET (n_distinct)" if n_distinct&.zero?),
+        *("#{alter} SET STORAGE #{storage.to_s.upcase}" if storage.present?),
+      ]
+    end
+  end
+end

data/lib/pg_trunk/operations/materialized_views/create_materialized_view.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: false
+
+# @!method ActiveRecord::Migration#create_materialized_view(name, **options, &block)
+# Create a materialized view
+#
+# @param [#to_s] name (nil) The qualified name of the view
+# @option [Boolean] :if_not_exists (false) Suppress the error when a view has been already created
+# @option [#to_s] :sql_definition (nil) The snippet containing the query
+# @option [#to_i] :version (nil)
+#   The alternative way to set sql_definition by referencing to a file containing the snippet
+# @option [#to_s] :tablespace (nil) The tablespace for the view
+# @option [Boolean] :with_data (true) If the view should be populated after creation
+# @option [#to_s] :comment (nil) The comment describing the view
+# @yield [Proc] the block with the view's definition
+# @yieldparam The receiver of methods specifying the view
+#
+# The operation creates the view using its `sql_definition`:
+#
+#   create_materialized_view("views.admin_users", sql_definition: <<~SQL)
+#     SELECT id, name FROM users WHERE admin;
+#   SQL
+#
+# For compatibility to the `scenic` gem, we also support
+# adding a definition via its version:
+#
+#    create_materialized_view "admin_users", version: 1
+#
+# It is expected, that a `db/materialized_views/admin_users_v01.sql`
+# to contain the SQL snippet.
+#
+# The tablespace can be specified for the created view.
+# Notice that later it can't be changed (in PostgreSQL all rows
+# can be moved to another tablespace, but we don't support
+# this feature yet).
+#
+#   create_materialized_view "admin_users" do |v|
+#     v.tablespace "fast_ssd"
+#     v.sql_definition <<~SQL
+#       SELECT id, name, password, admin, on_duty
+#       FROM users
+#       WHERE admin
+#     SQL
+#   end
+#
+# You can also set a comment describing the view,
+# and redefine the storage options for some TOAST-ed columns,
+# as well as their custom statistics:
+#
+#   create_materialized_view "admin_users" do |v|
+#     v.sql_definition <<~SQL
+#       SELECT id, name, password, admin, on_duty
+#       FROM users
+#       WHERE admin
+#     SQL
+#
+#     v.column "password", storage: "external" # to avoid compression
+#     v.column "password", n_distinct: -1 # linear dependency
+#     v.column "admin", n_distinct: 1 # exact number of values
+#     v.column "on_duty", statistics: 2 # the total number of values
+#
+#     v.comment "Admin users only"
+#   end
+#
+# With the `replace_existing: true` option the operation
+# would use `CREATE OR REPLACE VIEW` command, so it
+# can be used to "update" (or reload) the existing view.
+#
+#   create_materialized_view "admin_users",
+#                            version: 1,
+#                            replace_existing: true
+#
+# This option makes the migration irreversible due to uncertainty
+# of the previous state of the database.
+
+module PGTrunk::Operations::MaterializedViews
+  # @private
+  class CreateMaterializedView < Base
+    validates :sql_definition, presence: true
+    # Forbid these attributes
+    validates :algorithm, :cluster_on, :force, :if_exists, :new_name, absence: true
+
+    from_sql do |_version|
+      <<~SQL
+        SELECT
+          c.oid,
+          (c.relnamespace::regnamespace || '.' || c.relname) AS name,
+          t.spcname AS "tablespace",
+          replace(pg_get_viewdef(c.oid, 60), ';', '') AS sql_definition,
+          (CASE WHEN NOT m.ispopulated THEN false END) AS with_data,
+          (
+            SELECT
+              json_agg(
+                json_build_object(
+                  'name', a.attname,
+                  'storage', (
+                    CASE
+                      WHEN a.attstorage = 'p' THEN 'plain'
+                      WHEN a.attstorage = 'e' THEN 'external'
+                      WHEN a.attstorage = 'x' THEN 'extended'
+                      WHEN a.attstorage = 'm' THEN 'main'
+                    END
+                  )
+                ) ORDER BY a.attnum
+              )
+            FROM pg_attribute a LEFT JOIN pg_type t ON t.oid = a.atttypid
+            WHERE c.oid = a.attrelid AND t.typstorage != a.attstorage
+          ) AS "columns",
+          d.description AS comment
+        FROM pg_class c
+          JOIN pg_trunk e ON e.oid = c.oid AND e.classid = 'pg_class'::regclass
+          JOIN pg_matviews m ON m.matviewname = c.relname
+            AND m.schemaname::regnamespace = c.relnamespace::regnamespace
+          LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
+          LEFT JOIN pg_description d ON d.objoid = c.oid
+            AND d.classoid = 'pg_class'::regclass
+        WHERE c.relkind = 'm';
+      SQL
+    end
+
+    def to_sql(_version)
+      [create_view, *alter_columns, *create_comment, register_view].join(" ")
+    end
+
+    def invert
+      irreversible!("if_not_exists: true") if if_not_exists
+      DropMaterializedView.new(name: name)
+    end
+
+    private
+
+    def create_view
+      sql = "CREATE MATERIALIZED VIEW"
+      sql << " IF NOT EXISTS" if if_not_exists
+      sql << " #{name.to_sql}"
+      sql << " TABLESPACE #{tablespace.inspect}" if tablespace.present?
+      sql << " AS #{sql_definition}"
+      sql << " WITH NO DATA" if with_data == false
+      sql << ";"
+    end
+
+    def alter_columns
+      return if columns.blank?
+
+      sql = "ALTER MATERIALIZED VIEW #{name.to_sql}"
+      sql << columns.flat_map(&:to_sql).join(", ")
+      sql << ";"
+    end
+
+    def create_comment
+      return if comment.blank?
+
+      <<~SQL
+        COMMENT ON MATERIALIZED VIEW #{name.to_sql}
+        IS $comment$#{comment}$comment$;
+      SQL
+    end
+
+    def register_view
+      <<~SQL.squish
+        INSERT INTO pg_trunk (oid, classid)
+          SELECT oid, 'pg_class'::regclass
+          FROM pg_class
+          WHERE relname = #{name.quoted}
+            AND relnamespace = #{name.namespace}
+            AND relkind = 'm'
+        ON CONFLICT DO NOTHING;
+      SQL
+    end
+  end
+end

data/lib/pg_trunk/operations/materialized_views/drop_materialized_view.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: false
+
+# @!method ActiveRecord::Migration#drop_materialized_view(name, **options, &block)
+# Drop a materialized view
+#
+# @param [#to_s] name (nil) The qualified name of the view
+# @option [Boolean] :if_exists (false) Suppress the error when the view is absent
+# @option [Symbol] :force (:restrict) How to process dependent objects (`:cascade` or `:restrict`)
+# @option [#to_s] :sql_definition (nil) The snippet containing the query
+# @option [#to_i] :revert_to_version (nil)
+#   The alternative way to set sql_definition by referencing to a file containing the snippet
+# @option [#to_s] :tablespace (nil) The tablespace for the view
+# @option [Boolean] :with_data (true) If the view should be populated after creation
+# @option [#to_s] :comment (nil) The comment describing the view
+# @yield [Proc] the block with the view's definition
+# @yieldparam The receiver of methods specifying the view
+#
+# The operation drops a materialized view identified by its
+# qualified name (it can include a schema).
+#
+#   drop_materialized_view "views.admin_users"
+#
+# To make the operation invertible, use the same options
+# as in the `create_view` operation.
+#
+#   drop_materialized_view "views.admin_users" do |v|
+#     v.sql_definition "SELECT name, password FROM users WHERE admin;"
+#     v.column "password", storage: "external" # prevent compression
+#     v.with_data false
+#     v.comment "Admin users only"
+#   end
+#
+# You can also use a version-base SQL definition like:
+#
+#    drop_materialized_view "admin_users", revert_to_version: 1
+#
+# With the `force: :cascade` option the operation would remove
+# all the objects which depend on the view.
+#
+#   drop_materialized_view "admin_users", force: :cascade
+#
+# With the `if_exists: true` option the operation won't fail
+# even when the view was absent in the database.
+#
+#   drop_materialized_view "admin_users", if_exists: true
+#
+# Both options make a migration irreversible due to uncertainty
+# of the previous state of the database.
+
+module PGTrunk::Operations::MaterializedViews
+  # @private
+  class DropMaterializedView < Base
+    # Forbid these attributes
+    validates :algorithm, :cluster_on, :if_not_exists, :new_name, absence: true
+
+    def to_sql(_version)
+      sql = "DROP MATERIALIZED VIEW"
+      sql << " IF EXISTS" if if_exists
+      sql << " #{name.to_sql}"
+      sql << " CASCADE" if force == :cascade
+      sql << ";"
+    end
+
+    def invert
+      irreversible!("if_exists: true") if if_exists
+      irreversible!("force: :cascade") if force == :cascade
+      CreateMaterializedView.new(**to_h.except(:force))
+    end
+  end
+end

data/lib/pg_trunk/operations/materialized_views/refresh_materialized_view.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: false
+
+# @!method ActiveRecord::Migration#refresh_materialized_view(name, **options)
+# Refresh a materialized view
+#
+# @param [#to_s] name (nil) The qualified name of the view
+# @option [Boolean] :with_data (true) If the view should be populated after creation
+# @option [Symbol] :algorithm (nil) Makes the operation concurrent when set to :concurrently
+#
+# The operation enables refreshing a materialized view
+# by reloading its underlying SQL query:
+#
+#   refresh_materialized_view "admin_users"
+#
+# The option `algorithm: :concurrently` acts exactly
+# like in the `create_index` definition. You should
+# possibly add the `disable_ddl_transaction!` command
+# to the migration as well.
+#
+# With option `with_data: false` the command won't
+# update the data. This option can't be used along with
+# the `:algorithm`.
+#
+# The operation is always reversible, though its
+# inversion does nothing.
+
+module PGTrunk::Operations::MaterializedViews
+  # @private
+  class RefreshMaterializedView < Base
+    validate do
+      errors.add :algorithm, :present if with_data == false && algorithm
+    end
+    validates :cluster_on, :columns, :force, :if_exists, :if_not_exists,
+              :new_name, :sql_definition, :tablespace, :version, :comment,
+              absence: true
+
+    def to_sql(_version)
+      sql = "REFRESH MATERIALIZED VIEW"
+      sql << " CONCURRENTLY" if algorithm == :concurrently
+      sql << " #{name.to_sql}"
+      sql << " WITH NO DATA" if with_data == false
+      sql << ";"
+    end
+
+    # The operation is reversible but its inversion does nothing
+    def invert; end
+  end
+end