scenic 1.7.0 → 1.9.0

data/lib/generators/scenic/view/view_generator.rb

@@ -28,12 +28,12 @@ module Scenic
         if creating_new_view? || destroying_initial_view?
           migration_template(
             "db/migrate/create_view.erb",
-            "db/migrate/create_#{plural_file_name}.rb",
+            "db/migrate/create_#{plural_file_name}.rb"
           )
         else
           migration_template(
             "db/migrate/update_view.erb",
-            "db/migrate/update_#{plural_file_name}_to_version_#{version}.rb",
+            "db/migrate/update_#{plural_file_name}_to_version_#{version}.rb"
           )
         end
       end
@@ -56,7 +56,7 @@ module Scenic
 
         def migration_class_name
           if creating_new_view?
-            "Create#{class_name.tr('.', '').pluralize}"
+            "Create#{class_name.tr(".", "").pluralize}"
           else
             "Update#{class_name.pluralize}ToVersion#{version}"
           end
@@ -73,7 +73,7 @@ module Scenic
 
       private
 
-      alias singular_name file_name
+      alias_method :singular_name, :file_name
 
       def file_name
         super.tr(".", "_")
@@ -113,7 +113,7 @@ module Scenic
 
       def create_view_options
         if materialized?
-          ", materialized: #{no_data? ? '{ no_data: true }' : true}"
+          ", materialized: #{no_data? ? "{ no_data: true }" : true}"
         else
           ""
         end

data/lib/scenic/adapters/postgres/index_creation.rb

@@ -0,0 +1,68 @@
+module Scenic
+  module Adapters
+    class Postgres
+      # Used to resiliently create indexes on a materialized view. If the index
+      # cannot be applied to the view (e.g. the columns don't exist any longer),
+      # we log that information and continue rather than raising an error. It is
+      # left to the user to judge whether the index is necessary and recreate
+      # it.
+      #
+      # Used when updating a materialized view to ensure the new version has all
+      # apprioriate indexes.
+      #
+      # @api private
+      class IndexCreation
+        # Creates the index creation object.
+        #
+        # @param connection [Connection] The connection to execute SQL against.
+        # @param speaker [#say] (ActiveRecord::Migration) The object used for
+        #   logging the results of creating indexes.
+        def initialize(connection:, speaker: ActiveRecord::Migration.new)
+          @connection = connection
+          @speaker = speaker
+        end
+
+        # Creates the provided indexes. If an index cannot be created, it is
+        # logged and the process continues.
+        #
+        # @param indexes [Array<Scenic::Index>] The indexes to create.
+        #
+        # @return [void]
+        def try_create(indexes)
+          Array(indexes).each(&method(:try_index_create))
+        end
+
+        private
+
+        attr_reader :connection, :speaker
+
+        def try_index_create(index)
+          success = with_savepoint(index.index_name) do
+            connection.execute(index.definition)
+          end
+
+          if success
+            say "index '#{index.index_name}' on '#{index.object_name}' has been created"
+          else
+            say "index '#{index.index_name}' on '#{index.object_name}' is no longer valid and has been dropped."
+          end
+        end
+
+        def with_savepoint(name)
+          connection.execute("SAVEPOINT #{name}")
+          yield
+          connection.execute("RELEASE SAVEPOINT #{name}")
+          true
+        rescue
+          connection.execute("ROLLBACK TO SAVEPOINT #{name}")
+          false
+        end
+
+        def say(message)
+          subitem = true
+          speaker.say(message, subitem)
+        end
+      end
+    end
+  end
+end

data/lib/scenic/adapters/postgres/index_migration.rb

@@ -0,0 +1,70 @@
+module Scenic
+  module Adapters
+    class Postgres
+      # Used during side-by-side materialized view updates to migrate indexes
+      # from the original view to the new view.
+      #
+      # @api private
+      class IndexMigration
+        # Creates the index migration object.
+        #
+        # @param connection [Connection] The connection to execute SQL against.
+        # @param speaker [#say] (ActiveRecord::Migration) The object used for
+        #   logging the results of migrating indexes.
+        def initialize(connection:, speaker: ActiveRecord::Migration.new)
+          @connection = connection
+          @speaker = speaker
+        end
+
+        # Retreives the indexes on the original view, renames them to avoid
+        # collisions, retargets the indexes to the destination view, and then
+        # creates the retargeted indexes.
+        #
+        # @param from [String] The name of the original view.
+        # @param to [String] The name of the destination view.
+        #
+        # @return [void]
+        def migrate(from:, to:)
+          source_indexes = Indexes.new(connection: connection).on(from)
+          retargeted_indexes = source_indexes.map { |i| retarget(i, to: to) }
+          source_indexes.each(&method(:rename))
+
+          if source_indexes.any?
+            say "indexes on '#{from}' have been renamed to avoid collisions"
+          end
+
+          IndexCreation
+            .new(connection: connection, speaker: speaker)
+            .try_create(retargeted_indexes)
+        end
+
+        private
+
+        attr_reader :connection, :speaker
+
+        def retarget(index, to:)
+          new_definition = index.definition.sub(
+            /ON (.*)\.#{index.object_name}/,
+            'ON \1.' + to + " "
+          )
+
+          Scenic::Index.new(
+            object_name: to,
+            index_name: index.index_name,
+            definition: new_definition
+          )
+        end
+
+        def rename(index)
+          temporary_name = TemporaryName.new(index.index_name).to_s
+          connection.rename_index(index.object_name, index.index_name, temporary_name)
+        end
+
+        def say(message)
+          subitem = true
+          speaker.say(message, subitem)
+        end
+      end
+    end
+  end
+end

data/lib/scenic/adapters/postgres/index_reapplication.rb

@@ -32,39 +32,14 @@ module Scenic
 
           yield
 
-          indexes.each(&method(:try_index_create))
+          IndexCreation
+            .new(connection: connection, speaker: speaker)
+            .try_create(indexes)
         end
 
         private
 
         attr_reader :connection, :speaker
-
-        def try_index_create(index)
-          success = with_savepoint(index.index_name) do
-            connection.execute(index.definition)
-          end
-
-          if success
-            say "index '#{index.index_name}' on '#{index.object_name}' has been recreated"
-          else
-            say "index '#{index.index_name}' on '#{index.object_name}' is no longer valid and has been dropped."
-          end
-        end
-
-        def with_savepoint(name)
-          connection.execute("SAVEPOINT #{name}")
-          yield
-          connection.execute("RELEASE SAVEPOINT #{name}")
-          true
-        rescue
-          connection.execute("ROLLBACK TO SAVEPOINT #{name}")
-          false
-        end
-
-        def say(message)
-          subitem = true
-          speaker.say(message, subitem)
-        end
       end
     end
   end

data/lib/scenic/adapters/postgres/indexes.rb

@@ -44,7 +44,7 @@ module Scenic
           Scenic::Index.new(
             object_name: result["object_name"],
             index_name: result["index_name"],
-            definition: result["definition"],
+            definition: result["definition"]
           )
         end
       end

data/lib/scenic/adapters/postgres/refresh_dependencies.rb

@@ -17,7 +17,7 @@ module Scenic
           dependencies.each do |dependency|
             adapter.refresh_materialized_view(
               dependency,
-              concurrently: concurrently,
+              concurrently: concurrently
             )
           end
         end
@@ -103,8 +103,8 @@ module Scenic
           ORDER BY class_for_rewrite.relname;
         SQL
 
-        private_constant "DependencyParser"
-        private_constant "DEPENDENCY_SQL"
+        private_constant :DependencyParser
+        private_constant :DEPENDENCY_SQL
 
         def dependencies
           raw_dependency_info = connection.select_rows(DEPENDENCY_SQL)

data/lib/scenic/adapters/postgres/side_by_side.rb

@@ -0,0 +1,50 @@
+module Scenic
+  module Adapters
+    class Postgres
+      # Updates a view using the `side-by-side` strategy where the new view is
+      # created and populated under a temporary name before the existing view is
+      # dropped and the temporary view is renamed to the original name.
+      class SideBySide
+        def initialize(adapter:, name:, definition:, speaker: ActiveRecord::Migration.new)
+          @adapter = adapter
+          @name = name
+          @definition = definition
+          @temporary_name = TemporaryName.new(name).to_s
+          @speaker = speaker
+        end
+
+        def update
+          adapter.create_materialized_view(temporary_name, definition)
+          say "temporary materialized view '#{temporary_name}' has been created"
+
+          IndexMigration
+            .new(connection: adapter.connection, speaker: speaker)
+            .migrate(from: name, to: temporary_name)
+
+          adapter.drop_materialized_view(name)
+          say "materialized view '#{name}' has been dropped"
+
+          rename_materialized_view(temporary_name, name)
+          say "temporary materialized view '#{temporary_name}' has been renamed to '#{name}'"
+        end
+
+        private
+
+        attr_reader :adapter, :name, :definition, :temporary_name, :speaker
+
+        def connection
+          adapter.connection
+        end
+
+        def rename_materialized_view(from, to)
+          connection.execute("ALTER MATERIALIZED VIEW #{from} RENAME TO #{to}")
+        end
+
+        def say(message)
+          subitem = true
+          speaker.say(message, subitem)
+        end
+      end
+    end
+  end
+end

data/lib/scenic/adapters/postgres/temporary_name.rb

@@ -0,0 +1,34 @@
+module Scenic
+  module Adapters
+    class Postgres
+      # Generates a temporary object name used internally by Scenic. This is
+      # used during side-by-side materialized view updates to avoid naming
+      # collisions. The generated name is based on a SHA1 hash of the original
+      # which ensures we do not exceed the 63 character limit for object names.
+      #
+      # @api private
+      class TemporaryName
+        # The prefix used for all temporary names.
+        PREFIX = "_scenic_sbs_".freeze
+
+        # Creates a new temporary name object.
+        #
+        # @param name [String] The original name to base the temporary name on.
+        def initialize(name)
+          @name = name
+          @salt = SecureRandom.hex(4)
+          @temporary_name = "#{PREFIX}#{Digest::SHA1.hexdigest(name + salt)}"
+        end
+
+        # @return [String] The temporary name.
+        def to_s
+          temporary_name
+        end
+
+        private
+
+        attr_reader :name, :temporary_name, :salt
+      end
+    end
+  end
+end

data/lib/scenic/adapters/postgres/views.rb

@@ -8,20 +8,91 @@ module Scenic
           @connection = connection
         end
 
-        # All of the views that this connection has defined.
+        # All of the views that this connection has defined, sorted according to
+        # dependencies between the views to facilitate dumping and loading.
         #
         # This will include materialized views if those are supported by the
         # connection.
         #
         # @return [Array<Scenic::View>]
         def all
-          views_from_postgres.map(&method(:to_scenic_view))
+          scenic_views = views_from_postgres.map(&method(:to_scenic_view))
+          sort(scenic_views)
         end
 
         private
 
+        def sort(scenic_views)
+          scenic_view_names = scenic_views.map(&:name)
+
+          tsorted_views(scenic_view_names).map do |view_name|
+            scenic_views.find do |sv|
+              sv.name == view_name || sv.name == view_name.split(".").last
+            end
+          end.compact
+        end
+
+        # When dumping the views, their order must be topologically
+        # sorted to take into account dependencies
+        def tsorted_views(views_names)
+          views_hash = TSortableHash.new
+
+          ::Scenic.database.execute(DEPENDENT_SQL).each do |relation|
+            source_v = [
+              relation["source_schema"],
+              relation["source_table"]
+            ].compact.join(".")
+
+            dependent = [
+              relation["dependent_schema"],
+              relation["dependent_view"]
+            ].compact.join(".")
+
+            views_hash[dependent] ||= []
+            views_hash[source_v] ||= []
+            views_hash[dependent] << source_v
+
+            views_names.delete(relation["source_table"])
+            views_names.delete(relation["dependent_view"])
+          end
+
+          # after dependencies, there might be some views left
+          # that don't have any dependencies
+          views_names.sort.each { |v| views_hash[v] ||= [] }
+          views_hash.tsort
+        end
+
         attr_reader :connection
 
+        # Query for the dependencies between views
+        DEPENDENT_SQL = <<~SQL.freeze
+          SELECT distinct dependent_ns.nspname AS dependent_schema
+          , dependent_view.relname AS dependent_view
+          , source_ns.nspname AS source_schema
+          , source_table.relname AS source_table
+          FROM pg_depend
+          JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid
+          JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid
+          JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid
+          JOIN pg_namespace dependent_ns ON dependent_ns.oid = dependent_view.relnamespace
+          JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace
+          WHERE dependent_ns.nspname = ANY (current_schemas(false)) AND source_ns.nspname = ANY (current_schemas(false))
+          AND source_table.relname != dependent_view.relname
+          AND source_table.relkind IN ('m', 'v') AND dependent_view.relkind IN ('m', 'v')
+          ORDER BY dependent_view.relname;
+        SQL
+        private_constant :DEPENDENT_SQL
+
+        class TSortableHash < Hash
+          include TSort
+
+          alias_method :tsort_each_node, :each_key
+          def tsort_each_child(node, &)
+            fetch(node).each(&)
+          end
+        end
+        private_constant :TSortableHash
+
         def views_from_postgres
           connection.execute(<<-SQL)
             SELECT
@@ -41,23 +112,25 @@ module Scenic
         end
 
         def to_scenic_view(result)
-          namespace, viewname = result.values_at "namespace", "viewname"
+          Scenic::View.new(
+            name: namespaced_view_name(result),
+            definition: result["definition"].strip,
+            materialized: result["kind"] == "m"
+          )
+        end
+
+        def namespaced_view_name(result)
+          namespace, viewname = result.values_at("namespace", "viewname")
 
           if namespace != "public"
-            namespaced_viewname = "#{pg_identifier(namespace)}.#{pg_identifier(viewname)}"
+            "#{pg_identifier(namespace)}.#{pg_identifier(viewname)}"
           else
-            namespaced_viewname = pg_identifier(viewname)
+            pg_identifier(viewname)
           end
-
-          Scenic::View.new(
-            name: namespaced_viewname,
-            definition: result["definition"].strip,
-            materialized: result["kind"] == "m",
-          )
         end
 
         def pg_identifier(name)
-          return name if name =~ /^[a-zA-Z_][a-zA-Z0-9_]*$/
+          return name if /^[a-zA-Z_][a-zA-Z0-9_]*$/.match?(name)
 
           pgconn.quote_ident(name)
         end

data/lib/scenic/adapters/postgres.rb

@@ -4,6 +4,10 @@ require_relative "postgres/index_reapplication"
 require_relative "postgres/indexes"
 require_relative "postgres/views"
 require_relative "postgres/refresh_dependencies"
+require_relative "postgres/side_by_side"
+require_relative "postgres/index_creation"
+require_relative "postgres/index_migration"
+require_relative "postgres/temporary_name"
 
 module Scenic
   # Scenic database adapters.
@@ -14,7 +18,7 @@ module Scenic
   module Adapters
     # An adapter for managing Postgres views.
     #
-    # These methods are used interally by Scenic and are not intended for direct
+    # These methods are used internally by Scenic and are not intended for direct
     # use. Methods that alter database schema are intended to be called via
     # {Statements}, while {#refresh_materialized_view} is called via
     # {Scenic.database}.
@@ -124,7 +128,7 @@ module Scenic
       # @param sql_definition The SQL schema that defines the materialized view.
       # @param no_data [Boolean] Default: false. Set to true to create
       #   materialized view without running the associated query. You will need
-      #   to perform a non-concurrent refresh to populate with data.
+      #   to perform a refresh to populate with data.
       #
       # This is typically called in a migration via {Statements#create_view}.
       #
@@ -137,8 +141,8 @@ module Scenic
 
         execute <<-SQL
   CREATE MATERIALIZED VIEW #{quote_table_name(name)} AS
-  #{sql_definition.rstrip.chomp(';')}
-  #{'WITH NO DATA' if no_data};
+  #{sql_definition.rstrip.chomp(";")}
+  #{"WITH NO DATA" if no_data};
         SQL
       end
 
@@ -154,18 +158,27 @@ module Scenic
       # @param sql_definition The SQL schema for the updated view.
       # @param no_data [Boolean] Default: false. Set to true to create
       #   materialized view without running the associated query. You will need
-      #   to perform a non-concurrent refresh to populate with data.
+      #   to perform a refresh to populate with data.
+      # @param side_by_side [Boolean] Default: false. Set to true to create the
+      #   new version under a different name and atomically swap them, limiting
+      #   the time that a view is inaccessible at the cost of doubling disk usage
       #
       # @raise [MaterializedViewsNotSupportedError] if the version of Postgres
       #   in use does not support materialized views.
       #
       # @return [void]
-      def update_materialized_view(name, sql_definition, no_data: false)
+      def update_materialized_view(name, sql_definition, no_data: false, side_by_side: false)
         raise_unless_materialized_views_supported
 
-        IndexReapplication.new(connection: connection).on(name) do
-          drop_materialized_view(name)
-          create_materialized_view(name, sql_definition, no_data: no_data)
+        if side_by_side
+          SideBySide
+            .new(adapter: self, name: name, definition: sql_definition)
+            .update
+        else
+          IndexReapplication.new(connection: connection).on(name) do
+            drop_materialized_view(name)
+            create_materialized_view(name, sql_definition, no_data: no_data)
+          end
         end
       end
 
@@ -193,7 +206,10 @@ module Scenic
       #   refreshed without locking the view for select but requires that the
       #   table have at least one unique index that covers all rows. Attempts to
       #   refresh concurrently without a unique index will raise a descriptive
-      #   error.
+      #   error. This option is ignored if the view is not populated, as it
+      #   would cause an error to be raised by Postgres. Default: false.
+      # @param cascade [Boolean] Whether to refresh dependent materialized
+      #   views. Default: false.
       #
       # @raise [MaterializedViewsNotSupportedError] if the version of Postgres
       #   in use does not support materialized views.
@@ -205,32 +221,64 @@ module Scenic
       #   Scenic.database.refresh_materialized_view(:search_results)
       # @example Concurrent refresh
       #   Scenic.database.refresh_materialized_view(:posts, concurrently: true)
+      # @example Cascade refresh
+      #   Scenic.database.refresh_materialized_view(:posts, cascade: true)
       #
       # @return [void]
       def refresh_materialized_view(name, concurrently: false, cascade: false)
         raise_unless_materialized_views_supported
 
+        if concurrently
+          raise_unless_concurrent_refresh_supported
+        end
+
         if cascade
           refresh_dependencies_for(name, concurrently: concurrently)
         end
 
-        if concurrently
-          raise_unless_concurrent_refresh_supported
+        if concurrently && populated?(name)
           execute "REFRESH MATERIALIZED VIEW CONCURRENTLY #{quote_table_name(name)};"
         else
           execute "REFRESH MATERIALIZED VIEW #{quote_table_name(name)};"
         end
       end
 
-      private
+      # True if supplied relation name is populated.
+      #
+      # @param name The name of the relation
+      #
+      # @raise [MaterializedViewsNotSupportedError] if the version of Postgres
+      #   in use does not support materialized views.
+      #
+      # @return [boolean]
+      def populated?(name)
+        raise_unless_materialized_views_supported
 
-      attr_reader :connectable
-      delegate :execute, :quote_table_name, to: :connection
+        schemaless_name = name.to_s.split(".").last
+
+        sql = "SELECT relispopulated FROM pg_class WHERE relname = '#{schemaless_name}'"
+        relations = execute(sql)
+
+        if relations.count.positive?
+          relations.first["relispopulated"].in?(["t", true])
+        else
+          false
+        end
+      end
 
+      # A decorated ActiveRecord connection object with some Scenic-specific
+      # methods. Not intended for direct use outside of the Postgres adapter.
+      #
+      # @api private
       def connection
         Connection.new(connectable.connection)
       end
 
+      private
+
+      attr_reader :connectable
+      delegate :execute, :quote_table_name, to: :connection
+
       def raise_unless_materialized_views_supported
         unless connection.supports_materialized_views?
           raise MaterializedViewsNotSupportedError
@@ -248,7 +296,7 @@ module Scenic
           name,
           self,
           connection,
-          concurrently: concurrently,
+          concurrently: concurrently
         )
       end
     end

data/lib/scenic/definition.rb

@@ -31,7 +31,7 @@ module Scenic
     attr_reader :name
 
     def filename
-      "#{UnaffixedName.for(name).tr('.', '_')}_v#{version}.sql"
+      "#{UnaffixedName.for(name).tr(".", "_")}_v#{version}.sql"
     end
   end
 end

data/lib/scenic/schema_dumper.rb

@@ -26,19 +26,5 @@ module Scenic
         ignored?(view.name)
       end
     end
-
-    unless ActiveRecord::SchemaDumper.private_instance_methods(false).include?(:ignored?)
-      # This method will be present in Rails 4.2.0 and can be removed then.
-      def ignored?(table_name)
-        ["schema_migrations", ignore_tables].flatten.any? do |ignored|
-          case ignored
-          when String then remove_prefix_and_suffix(table_name) == ignored
-          when Regexp then remove_prefix_and_suffix(table_name) =~ ignored
-          else
-            raise StandardError, "ActiveRecord::SchemaDumper.ignore_tables accepts an array of String and / or Regexp values."
-          end
-        end
-      end
-    end
   end
 end