scenic 1.8.0 → 1.9.0

data/lib/scenic/adapters/postgres/index_reapplication.rb

@@ -32,39 +32,14 @@ module Scenic
 
           yield
 
-          indexes.each(&method(:try_index_create))
+          IndexCreation
+            .new(connection: connection, speaker: speaker)
+            .try_create(indexes)
         end
 
         private
 
         attr_reader :connection, :speaker
-
-        def try_index_create(index)
-          success = with_savepoint(index.index_name) do
-            connection.execute(index.definition)
-          end
-
-          if success
-            say "index '#{index.index_name}' on '#{index.object_name}' has been recreated"
-          else
-            say "index '#{index.index_name}' on '#{index.object_name}' is no longer valid and has been dropped."
-          end
-        end
-
-        def with_savepoint(name)
-          connection.execute("SAVEPOINT #{name}")
-          yield
-          connection.execute("RELEASE SAVEPOINT #{name}")
-          true
-        rescue
-          connection.execute("ROLLBACK TO SAVEPOINT #{name}")
-          false
-        end
-
-        def say(message)
-          subitem = true
-          speaker.say(message, subitem)
-        end
       end
     end
   end

data/lib/scenic/adapters/postgres/side_by_side.rb

@@ -0,0 +1,50 @@
+module Scenic
+  module Adapters
+    class Postgres
+      # Updates a view using the `side-by-side` strategy where the new view is
+      # created and populated under a temporary name before the existing view is
+      # dropped and the temporary view is renamed to the original name.
+      class SideBySide
+        def initialize(adapter:, name:, definition:, speaker: ActiveRecord::Migration.new)
+          @adapter = adapter
+          @name = name
+          @definition = definition
+          @temporary_name = TemporaryName.new(name).to_s
+          @speaker = speaker
+        end
+
+        def update
+          adapter.create_materialized_view(temporary_name, definition)
+          say "temporary materialized view '#{temporary_name}' has been created"
+
+          IndexMigration
+            .new(connection: adapter.connection, speaker: speaker)
+            .migrate(from: name, to: temporary_name)
+
+          adapter.drop_materialized_view(name)
+          say "materialized view '#{name}' has been dropped"
+
+          rename_materialized_view(temporary_name, name)
+          say "temporary materialized view '#{temporary_name}' has been renamed to '#{name}'"
+        end
+
+        private
+
+        attr_reader :adapter, :name, :definition, :temporary_name, :speaker
+
+        def connection
+          adapter.connection
+        end
+
+        def rename_materialized_view(from, to)
+          connection.execute("ALTER MATERIALIZED VIEW #{from} RENAME TO #{to}")
+        end
+
+        def say(message)
+          subitem = true
+          speaker.say(message, subitem)
+        end
+      end
+    end
+  end
+end

data/lib/scenic/adapters/postgres/temporary_name.rb

@@ -0,0 +1,34 @@
+module Scenic
+  module Adapters
+    class Postgres
+      # Generates a temporary object name used internally by Scenic. This is
+      # used during side-by-side materialized view updates to avoid naming
+      # collisions. The generated name is based on a SHA1 hash of the original
+      # which ensures we do not exceed the 63 character limit for object names.
+      #
+      # @api private
+      class TemporaryName
+        # The prefix used for all temporary names.
+        PREFIX = "_scenic_sbs_".freeze
+
+        # Creates a new temporary name object.
+        #
+        # @param name [String] The original name to base the temporary name on.
+        def initialize(name)
+          @name = name
+          @salt = SecureRandom.hex(4)
+          @temporary_name = "#{PREFIX}#{Digest::SHA1.hexdigest(name + salt)}"
+        end
+
+        # @return [String] The temporary name.
+        def to_s
+          temporary_name
+        end
+
+        private
+
+        attr_reader :name, :temporary_name, :salt
+      end
+    end
+  end
+end

data/lib/scenic/adapters/postgres/views.rb

@@ -8,20 +8,91 @@ module Scenic
           @connection = connection
         end
 
-        # All of the views that this connection has defined.
+        # All of the views that this connection has defined, sorted according to
+        # dependencies between the views to facilitate dumping and loading.
         #
         # This will include materialized views if those are supported by the
         # connection.
         #
         # @return [Array<Scenic::View>]
         def all
-          views_from_postgres.map(&method(:to_scenic_view))
+          scenic_views = views_from_postgres.map(&method(:to_scenic_view))
+          sort(scenic_views)
         end
 
         private
 
+        def sort(scenic_views)
+          scenic_view_names = scenic_views.map(&:name)
+
+          tsorted_views(scenic_view_names).map do |view_name|
+            scenic_views.find do |sv|
+              sv.name == view_name || sv.name == view_name.split(".").last
+            end
+          end.compact
+        end
+
+        # When dumping the views, their order must be topologically
+        # sorted to take into account dependencies
+        def tsorted_views(views_names)
+          views_hash = TSortableHash.new
+
+          ::Scenic.database.execute(DEPENDENT_SQL).each do |relation|
+            source_v = [
+              relation["source_schema"],
+              relation["source_table"]
+            ].compact.join(".")
+
+            dependent = [
+              relation["dependent_schema"],
+              relation["dependent_view"]
+            ].compact.join(".")
+
+            views_hash[dependent] ||= []
+            views_hash[source_v] ||= []
+            views_hash[dependent] << source_v
+
+            views_names.delete(relation["source_table"])
+            views_names.delete(relation["dependent_view"])
+          end
+
+          # after dependencies, there might be some views left
+          # that don't have any dependencies
+          views_names.sort.each { |v| views_hash[v] ||= [] }
+          views_hash.tsort
+        end
+
         attr_reader :connection
 
+        # Query for the dependencies between views
+        DEPENDENT_SQL = <<~SQL.freeze
+          SELECT distinct dependent_ns.nspname AS dependent_schema
+          , dependent_view.relname AS dependent_view
+          , source_ns.nspname AS source_schema
+          , source_table.relname AS source_table
+          FROM pg_depend
+          JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid
+          JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid
+          JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid
+          JOIN pg_namespace dependent_ns ON dependent_ns.oid = dependent_view.relnamespace
+          JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace
+          WHERE dependent_ns.nspname = ANY (current_schemas(false)) AND source_ns.nspname = ANY (current_schemas(false))
+          AND source_table.relname != dependent_view.relname
+          AND source_table.relkind IN ('m', 'v') AND dependent_view.relkind IN ('m', 'v')
+          ORDER BY dependent_view.relname;
+        SQL
+        private_constant :DEPENDENT_SQL
+
+        class TSortableHash < Hash
+          include TSort
+
+          alias_method :tsort_each_node, :each_key
+          def tsort_each_child(node, &)
+            fetch(node).each(&)
+          end
+        end
+        private_constant :TSortableHash
+
         def views_from_postgres
           connection.execute(<<-SQL)
             SELECT
@@ -41,19 +112,21 @@ module Scenic
         end
 
         def to_scenic_view(result)
-          namespace, viewname = result.values_at "namespace", "viewname"
+          Scenic::View.new(
+            name: namespaced_view_name(result),
+            definition: result["definition"].strip,
+            materialized: result["kind"] == "m"
+          )
+        end
 
-          namespaced_viewname = if namespace != "public"
+        def namespaced_view_name(result)
+          namespace, viewname = result.values_at("namespace", "viewname")
+
+          if namespace != "public"
             "#{pg_identifier(namespace)}.#{pg_identifier(viewname)}"
           else
             pg_identifier(viewname)
           end
-
-          Scenic::View.new(
-            name: namespaced_viewname,
-            definition: result["definition"].strip,
-            materialized: result["kind"] == "m"
-          )
         end
 
         def pg_identifier(name)

data/lib/scenic/adapters/postgres.rb

@@ -4,6 +4,10 @@ require_relative "postgres/index_reapplication"
 require_relative "postgres/indexes"
 require_relative "postgres/views"
 require_relative "postgres/refresh_dependencies"
+require_relative "postgres/side_by_side"
+require_relative "postgres/index_creation"
+require_relative "postgres/index_migration"
+require_relative "postgres/temporary_name"
 
 module Scenic
   # Scenic database adapters.
@@ -14,7 +18,7 @@ module Scenic
   module Adapters
     # An adapter for managing Postgres views.
     #
-    # These methods are used interally by Scenic and are not intended for direct
+    # These methods are used internally by Scenic and are not intended for direct
     # use. Methods that alter database schema are intended to be called via
     # {Statements}, while {#refresh_materialized_view} is called via
     # {Scenic.database}.
@@ -124,7 +128,7 @@ module Scenic
       # @param sql_definition The SQL schema that defines the materialized view.
       # @param no_data [Boolean] Default: false. Set to true to create
       #   materialized view without running the associated query. You will need
-      #   to perform a non-concurrent refresh to populate with data.
+      #   to perform a refresh to populate with data.
       #
       # This is typically called in a migration via {Statements#create_view}.
       #
@@ -154,18 +158,27 @@ module Scenic
       # @param sql_definition The SQL schema for the updated view.
       # @param no_data [Boolean] Default: false. Set to true to create
       #   materialized view without running the associated query. You will need
-      #   to perform a non-concurrent refresh to populate with data.
+      #   to perform a refresh to populate with data.
+      # @param side_by_side [Boolean] Default: false. Set to true to create the
+      #   new version under a different name and atomically swap them, limiting
+      #   the time that a view is inaccessible at the cost of doubling disk usage
       #
       # @raise [MaterializedViewsNotSupportedError] if the version of Postgres
       #   in use does not support materialized views.
       #
       # @return [void]
-      def update_materialized_view(name, sql_definition, no_data: false)
+      def update_materialized_view(name, sql_definition, no_data: false, side_by_side: false)
         raise_unless_materialized_views_supported
 
-        IndexReapplication.new(connection: connection).on(name) do
-          drop_materialized_view(name)
-          create_materialized_view(name, sql_definition, no_data: no_data)
+        if side_by_side
+          SideBySide
+            .new(adapter: self, name: name, definition: sql_definition)
+            .update
+        else
+          IndexReapplication.new(connection: connection).on(name) do
+            drop_materialized_view(name)
+            create_materialized_view(name, sql_definition, no_data: no_data)
+          end
         end
       end
 
@@ -193,7 +206,10 @@ module Scenic
       #   refreshed without locking the view for select but requires that the
       #   table have at least one unique index that covers all rows. Attempts to
       #   refresh concurrently without a unique index will raise a descriptive
-      #   error.
+      #   error. This option is ignored if the view is not populated, as it
+      #   would cause an error to be raised by Postgres. Default: false.
+      # @param cascade [Boolean] Whether to refresh dependent materialized
+      #   views. Default: false.
       #
       # @raise [MaterializedViewsNotSupportedError] if the version of Postgres
       #   in use does not support materialized views.
@@ -205,26 +221,29 @@ module Scenic
       #   Scenic.database.refresh_materialized_view(:search_results)
       # @example Concurrent refresh
       #   Scenic.database.refresh_materialized_view(:posts, concurrently: true)
+      # @example Cascade refresh
+      #   Scenic.database.refresh_materialized_view(:posts, cascade: true)
       #
       # @return [void]
       def refresh_materialized_view(name, concurrently: false, cascade: false)
         raise_unless_materialized_views_supported
 
+        if concurrently
+          raise_unless_concurrent_refresh_supported
+        end
+
         if cascade
           refresh_dependencies_for(name, concurrently: concurrently)
         end
 
-        if concurrently
-          raise_unless_concurrent_refresh_supported
+        if concurrently && populated?(name)
           execute "REFRESH MATERIALIZED VIEW CONCURRENTLY #{quote_table_name(name)};"
         else
           execute "REFRESH MATERIALIZED VIEW #{quote_table_name(name)};"
         end
       end
 
-      # True if supplied relation name is populated. Useful for checking the
-      # state of materialized views which may error if created `WITH NO DATA`
-      # and used before they are refreshed. True for all other relation types.
+      # True if supplied relation name is populated.
       #
       # @param name The name of the relation
       #
@@ -235,7 +254,7 @@ module Scenic
       def populated?(name)
         raise_unless_materialized_views_supported
 
-        schemaless_name = name.split(".").last
+        schemaless_name = name.to_s.split(".").last
 
         sql = "SELECT relispopulated FROM pg_class WHERE relname = '#{schemaless_name}'"
         relations = execute(sql)
@@ -247,15 +266,19 @@ module Scenic
         end
       end
 
+      # A decorated ActiveRecord connection object with some Scenic-specific
+      # methods. Not intended for direct use outside of the Postgres adapter.
+      #
+      # @api private
+      def connection
+        Connection.new(connectable.connection)
+      end
+
       private
 
       attr_reader :connectable
       delegate :execute, :quote_table_name, to: :connection
 
-      def connection
-        Connection.new(connectable.connection)
-      end
-
       def raise_unless_materialized_views_supported
         unless connection.supports_materialized_views?
           raise MaterializedViewsNotSupportedError

data/lib/scenic/schema_dumper.rb

@@ -26,19 +26,5 @@ module Scenic
         ignored?(view.name)
       end
     end
-
-    unless ActiveRecord::SchemaDumper.private_instance_methods(false).include?(:ignored?)
-      # This method will be present in Rails 4.2.0 and can be removed then.
-      def ignored?(table_name)
-        ["schema_migrations", ignore_tables].flatten.any? do |ignored|
-          case ignored
-          when String then remove_prefix_and_suffix(table_name) == ignored
-          when Regexp then remove_prefix_and_suffix(table_name) =~ ignored
-          else
-            raise StandardError, "ActiveRecord::SchemaDumper.ignore_tables accepts an array of String and / or Regexp values."
-          end
-        end
-      end
-    end
   end
 end

data/lib/scenic/statements.rb

@@ -9,9 +9,11 @@ module Scenic
     # @param sql_definition [String] The SQL query for the view schema. An error
     #   will be raised if `sql_definition` and `version` are both set,
     #   as they are mutually exclusive.
-    # @param materialized [Boolean, Hash] Set to true to create a materialized
-    #   view. Set to { no_data: true } to create materialized view without
-    #   loading data. Defaults to false.
+    # @param materialized [Boolean, Hash] Set to a truthy value to create a
+    #   materialized view. Hash
+    # @option materialized [Boolean] :no_data (false) Set to true to create
+    #   materialized view without running the associated query. You will need
+    #   to perform a non-concurrent refresh to populate with data.
     # @return The database response from executing the create statement.
     #
     # @example Create from `db/views/searches_v02.sql`
@@ -37,10 +39,12 @@ module Scenic
       sql_definition ||= definition(name, version)
 
       if materialized
+        options = materialized_options(materialized)
+
         Scenic.database.create_materialized_view(
           name,
           sql_definition,
-          no_data: no_data(materialized)
+          no_data: options[:no_data]
         )
       else
         Scenic.database.create_view(name, sql_definition)
@@ -80,14 +84,23 @@ module Scenic
     #   as they are mutually exclusive.
     # @param revert_to_version [Fixnum] The version number to rollback to on
     #   `rake db rollback`
-    # @param materialized [Boolean, Hash] True if updating a materialized view.
-    #   Set to { no_data: true } to update materialized view without loading
-    #   data. Defaults to false.
+    # @param materialized [Boolean, Hash] True or a Hash if updating a
+    #   materialized view.
+    # @option materialized [Boolean] :no_data (false) Set to true to update
+    #   a materialized view without loading data. You will need to perform a
+    #   refresh to populate with data. Cannot be combined with the :side_by_side
+    #   option.
+    # @option materialized [Boolean] :side_by_side (false) Set to true to update
+    #   update a materialized view using our side-by-side strategy, which will
+    #   limit the time the view is locked at the cost of increasing disk usage.
+    #   The view is initially updated with a temporary name and atomically
+    #   swapped once it is successfully created with data. Cannot be combined
+    #   with the :no_data option.
     # @return The database response from executing the create statement.
     #
     # @example
     #   update_view :engagement_reports, version: 3, revert_to_version: 2
-    #
+    #   update_view :comments, version: 2, revert_to_version: 1, materialized: { side_by_side: true }
     def update_view(name, version: nil, sql_definition: nil, revert_to_version: nil, materialized: false)
       if version.blank? && sql_definition.blank?
         raise(
@@ -106,10 +119,24 @@ module Scenic
       sql_definition ||= definition(name, version)
 
       if materialized
+        options = materialized_options(materialized)
+
+        if options[:no_data] && options[:side_by_side]
+          raise(
+            ArgumentError,
+            "no_data and side_by_side options cannot be combined"
+          )
+        end
+
+        if options[:side_by_side] && !transaction_open?
+          raise "a transaction is required to perform a side-by-side update"
+        end
+
         Scenic.database.update_materialized_view(
           name,
           sql_definition,
-          no_data: no_data(materialized)
+          no_data: options[:no_data],
+          side_by_side: options[:side_by_side]
         )
       else
         Scenic.database.update_view(name, sql_definition)
@@ -152,11 +179,17 @@ module Scenic
       Scenic::Definition.new(name, version).to_sql
     end
 
-    def no_data(materialized)
-      if materialized.is_a?(Hash)
-        materialized.fetch(:no_data, false)
+    def materialized_options(materialized)
+      if materialized.is_a? Hash
+        {
+          no_data: materialized.fetch(:no_data, false),
+          side_by_side: materialized.fetch(:side_by_side, false)
+        }
       else
-        false
+        {
+          no_data: false,
+          side_by_side: false
+        }
       end
     end
   end

data/lib/scenic/version.rb

@@ -1,3 +1,3 @@
 module Scenic
-  VERSION = "1.8.0".freeze
+  VERSION = "1.9.0".freeze
 end

data/scenic.gemspec

@@ -18,11 +18,15 @@ Gem::Specification.new do |spec|
   spec.files = `git ls-files -z`.split("\x0")
   spec.require_paths = ["lib"]
 
+  spec.metadata = {
+    "funding-uri" => "https://github.com/scenic-views/scenic"
+  }
+
   spec.add_development_dependency "bundler", ">= 1.5"
   spec.add_development_dependency "database_cleaner"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec", ">= 3.3"
-  spec.add_development_dependency "pg", "~> 0.19"
+  spec.add_development_dependency "pg"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "ammeter", ">= 1.1.3"
   spec.add_development_dependency "yard"

data/spec/acceptance/user_manages_views_spec.rb

@@ -45,6 +45,17 @@ describe "User manages views" do
     verify_result "Child.take.name", "Elliot"
     verify_schema_contains 'add_index "children"'
 
+    successfully "rails generate scenic:view child --materialized --side-by-side"
+    verify_identical_view_definitions "children_v02", "children_v03"
+
+    write_definition "children_v03", "SELECT 'Juniper'::text AS name"
+    successfully "rake db:migrate"
+
+    successfully "rake db:reset"
+    verify_result "Child.take.name", "Juniper"
+    verify_schema_contains 'add_index "children"'
+
+    successfully "rake db:rollback"
     successfully "rake db:rollback"
     successfully "rake db:rollback"
     successfully "rails destroy scenic:model child"

data/spec/dummy/config/application.rb

@@ -11,5 +11,9 @@ module Dummy
     config.cache_classes = true
     config.eager_load = false
     config.active_support.deprecation = :stderr
+
+    if config.active_support.respond_to?(:to_time_preserves_timezone)
+      config.active_support.to_time_preserves_timezone = :zone
+    end
   end
 end

data/spec/generators/scenic/view/view_generator_spec.rb

@@ -37,6 +37,32 @@ describe Scenic::Generators::ViewGenerator, :generator do
     end
   end
 
+  it "sets the no_data option when updating a materialized view" do
+    with_view_definition("aired_episodes", 1, "hello") do
+      allow(Dir).to receive(:entries).and_return(["aired_episodes_v01.sql"])
+
+      run_generator ["aired_episode", "--materialized", "--no-data"]
+      migration = migration_file(
+        "db/migrate/update_aired_episodes_to_version_2.rb"
+      )
+      expect(migration).to contain "materialized: { no_data: true }"
+      expect(migration).not_to contain "side_by_side"
+    end
+  end
+
+  it "sets the side-by-side option when updating a materialized view" do
+    with_view_definition("aired_episodes", 1, "hello") do
+      allow(Dir).to receive(:entries).and_return(["aired_episodes_v01.sql"])
+
+      run_generator ["aired_episode", "--materialized", "--side-by-side"]
+      migration = migration_file(
+        "db/migrate/update_aired_episodes_to_version_2.rb"
+      )
+      expect(migration).to contain "materialized: { side_by_side: true }"
+      expect(migration).not_to contain "no_data"
+    end
+  end
+
   it "uses 'replace_view' instead of 'update_view' if replace flag is set" do
     with_view_definition("aired_episodes", 1, "hello") do
       allow(Dir).to receive(:entries).and_return(["aired_episodes_v01.sql"])

data/spec/scenic/adapters/postgres/index_creation_spec.rb

@@ -0,0 +1,54 @@
+require "spec_helper"
+
+module Scenic
+  module Adapters
+    describe Postgres::IndexCreation, :db do
+      it "successfully recreates applicable indexes" do
+        create_materialized_view("hi", "SELECT 'hi' AS greeting")
+        speaker = DummySpeaker.new
+
+        index = Scenic::Index.new(
+          object_name: "hi",
+          index_name: "hi_greeting_idx",
+          definition: "CREATE INDEX hi_greeting_idx ON hi (greeting)"
+        )
+
+        Postgres::IndexCreation
+          .new(connection: ActiveRecord::Base.connection, speaker: speaker)
+          .try_create([index])
+
+        expect(indexes_for("hi")).not_to be_empty
+        expect(speaker.messages).to include(/index 'hi_greeting_idx' .* has been created/)
+      end
+
+      it "skips indexes that are not applicable" do
+        create_materialized_view("hi", "SELECT 'hi' AS greeting")
+        speaker = DummySpeaker.new
+        index = Scenic::Index.new(
+          object_name: "hi",
+          index_name: "hi_person_idx",
+          definition: "CREATE INDEX hi_person_idx ON hi (person)"
+        )
+
+        Postgres::IndexCreation
+          .new(connection: ActiveRecord::Base.connection, speaker: speaker)
+          .try_create([index])
+
+        expect(indexes_for("hi")).to be_empty
+        expect(speaker.messages).to include(/index 'hi_person_idx' .* has been dropped/)
+      end
+    end
+
+    class DummySpeaker
+      attr_reader :messages
+
+      def initialize
+        @messages = []
+      end
+
+      def say(message, bool = false)
+        @messages << message
+      end
+    end
+  end
+end