pg_online_schema_change 0.1.0 → 0.4.0

data/lib/pg_online_schema_change/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "thor"
 
 module PgOnlineSchemaChange
@@ -25,7 +27,7 @@ module PgOnlineSchemaChange
     def perform
       client_options = Struct.new(*options.keys.map(&:to_sym)).new(*options.values)
 
-      PgOnlineSchemaChange.logger = client_options.verbose
+      PgOnlineSchemaChange.logger(verbose: client_options.verbose)
       PgOnlineSchemaChange::Orchestrate.run!(client_options)
     end
 

data/lib/pg_online_schema_change/client.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "pg"
 
 module PgOnlineSchemaChange
@@ -16,7 +18,9 @@ module PgOnlineSchemaChange
       @drop = options.drop
       @kill_backends = options.kill_backends
       @wait_time_for_lock = options.wait_time_for_lock
+
       handle_copy_statement(options.copy_statement)
+      handle_validations
 
       @connection = PG.connect(
         dbname: @dbname,
@@ -26,17 +30,19 @@ module PgOnlineSchemaChange
         port: @port,
       )
 
-      raise Error, "Not a valid ALTER statement: #{@alter_statement}" unless Query.alter_statement?(@alter_statement)
-
-      unless Query.same_table?(@alter_statement)
-        raise Error "All statements should belong to the same table: #{@alter_statement}"
-      end
-
       @table = Query.table(@alter_statement)
 
       PgOnlineSchemaChange.logger.debug("Connection established")
     end
 
+    def handle_validations
+      raise Error, "Not a valid ALTER statement: #{@alter_statement}" unless Query.alter_statement?(@alter_statement)
+
+      return if Query.same_table?(@alter_statement)
+
+      raise Error "All statements should belong to the same table: #{@alter_statement}"
+    end
+
     def handle_copy_statement(statement)
       return if statement.nil? || statement == ""
 

data/lib/pg_online_schema_change/functions.rb

@@ -1,4 +1,6 @@
-FUNC_FIX_SERIAL_SEQUENCE = <<~SQL.freeze
+# frozen_string_literal: true
+
+FUNC_FIX_SERIAL_SEQUENCE = <<~SQL
     CREATE OR REPLACE FUNCTION fix_serial_sequence(_table regclass, _newtable text)
     RETURNS void AS
     $func$
@@ -35,7 +37,7 @@ FUNC_FIX_SERIAL_SEQUENCE = <<~SQL.freeze
   $func$  LANGUAGE plpgsql VOLATILE;
 SQL
 
-FUNC_CREATE_TABLE_ALL = <<~SQL.freeze
+FUNC_CREATE_TABLE_ALL = <<~SQL
   CREATE OR REPLACE FUNCTION create_table_all(source_table text, newsource_table text)
     RETURNS void language plpgsql
     as $$

data/lib/pg_online_schema_change/helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module PgOnlineSchemaChange
   module Helper
     def primary_key
@@ -15,7 +17,14 @@ module PgOnlineSchemaChange
       result = Store.send(:get, method)
       return result if result
 
-      raise ArgumentError, "Method `#{method}` doesn't exist."
+      super
+    end
+
+    def respond_to_missing?(method_name, *args)
+      result = Store.send(:get, method)
+      return true if result
+
+      super
     end
   end
 end

data/lib/pg_online_schema_change/orchestrate.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 require "securerandom"
 
 module PgOnlineSchemaChange
   class Orchestrate
-    SWAP_STATEMENT_TIMEOUT = "5s".freeze
+    SWAP_STATEMENT_TIMEOUT = "5s"
 
     extend Helper
 
@@ -21,12 +23,20 @@ module PgOnlineSchemaChange
         Query.run(client.connection, FUNC_FIX_SERIAL_SEQUENCE)
         Query.run(client.connection, FUNC_CREATE_TABLE_ALL)
 
+        setup_store
+      end
+
+      def setup_store
         # Set this early on to ensure their creation and cleanup (unexpected)
         # happens at all times. IOW, the calls from Store.get always return
         # the same value.
         Store.set(:old_primary_table, "pgosc_op_table_#{client.table}")
-        Store.set(:audit_table, "pgosc_at_#{client.table}_#{random_string}")
-        Store.set(:shadow_table, "pgosc_st_#{client.table}_#{random_string}")
+        Store.set(:audit_table, "pgosc_at_#{client.table}_#{pgosc_identifier}")
+        Store.set(:operation_type_column, "operation_type_#{pgosc_identifier}")
+        Store.set(:trigger_time_column, "trigger_time_#{pgosc_identifier}")
+        Store.set(:audit_table_pk, "at_#{pgosc_identifier}_id")
+        Store.set(:audit_table_pk_sequence, "#{audit_table}_#{audit_table_pk}_seq")
+        Store.set(:shadow_table, "pgosc_st_#{client.table}_#{pgosc_identifier}")
       end
 
       def run!(options)
@@ -37,10 +47,10 @@ module PgOnlineSchemaChange
 
         setup_audit_table!
         setup_trigger!
-        setup_shadow_table!
-        disable_vacuum!
-        run_alter_statement!
-        copy_data!
+        setup_shadow_table! # re-uses transaction with serializable
+        disable_vacuum! # re-uses transaction with serializable
+        run_alter_statement! # re-uses transaction with serializable
+        copy_data! # re-uses transaction with serializable
         run_analyze!
         replay_and_swap!
         run_analyze!
@@ -70,7 +80,7 @@ module PgOnlineSchemaChange
         reader = setup_signals!
         signal = reader.gets.chomp
 
-        while !reader.closed? && IO.select([reader])
+        while !reader.closed? && IO.select([reader]) # rubocop:disable Lint/UnreachableLoop
           logger.info "Signal #{signal} received, cleaning up"
 
           client.connection.cancel
@@ -85,7 +95,7 @@ module PgOnlineSchemaChange
         logger.info("Setting up audit table", { audit_table: audit_table })
 
         sql = <<~SQL
-          CREATE TABLE #{audit_table} (operation_type text, trigger_time timestamp, LIKE #{client.table});
+          CREATE TABLE #{audit_table} (#{audit_table_pk} SERIAL PRIMARY KEY, #{operation_type_column} text, #{trigger_time_column} timestamp, LIKE #{client.table});
         SQL
 
         Query.run(client.connection, sql)
@@ -109,13 +119,13 @@ module PgOnlineSchemaChange
           $$
           BEGIN
             IF ( TG_OP = 'INSERT') THEN
-              INSERT INTO \"#{audit_table}\" select 'INSERT', now(), NEW.* ;
+              INSERT INTO \"#{audit_table}\" select nextval(\'#{audit_table_pk_sequence}\'), 'INSERT', clock_timestamp(), NEW.* ;
               RETURN NEW;
             ELSIF ( TG_OP = 'UPDATE') THEN
-              INSERT INTO \"#{audit_table}\" select 'UPDATE', now(),  NEW.* ;
+              INSERT INTO \"#{audit_table}\" select nextval(\'#{audit_table_pk_sequence}\'), 'UPDATE', clock_timestamp(),  NEW.* ;
               RETURN NEW;
             ELSIF ( TG_OP = 'DELETE') THEN
-              INSERT INTO \"#{audit_table}\" select 'DELETE', now(), OLD.* ;
+              INSERT INTO \"#{audit_table}\" select nextval(\'#{audit_table_pk_sequence}\'), 'DELETE', clock_timestamp(), OLD.* ;
               RETURN NEW;
             END IF;
           END;
@@ -132,18 +142,28 @@ module PgOnlineSchemaChange
       end
 
       def setup_shadow_table!
+        # re-uses transaction with serializable
+        # This ensures that all queries from here till copy_data run with serializable.
+        # This is to to ensure that once the trigger is added to the primay table
+        # and contents being copied into the shadow, after a delete all on audit table,
+        # any replaying of rows that happen next from audit table do not contain
+        # any duplicates. We are ensuring there are no race conditions between
+        # adding the trigger, till the copy ends, since they all happen in the
+        # same serializable transaction.
+        Query.run(client.connection, "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE", true)
         logger.info("Setting up shadow table", { shadow_table: shadow_table })
 
-        Query.run(client.connection, "SELECT create_table_all('#{client.table}', '#{shadow_table}');")
+        Query.run(client.connection, "SELECT create_table_all('#{client.table}', '#{shadow_table}');", true)
 
         # update serials
-        Query.run(client.connection, "SELECT fix_serial_sequence('#{client.table}', '#{shadow_table}');")
+        Query.run(client.connection, "SELECT fix_serial_sequence('#{client.table}', '#{shadow_table}');", true)
       end
 
-      # Disabling vacuum to avoid any issues during the process
       def disable_vacuum!
-        result = Query.storage_parameters_for(client, client.table) || ""
-        primary_table_storage_parameters = Store.set(:primary_table_storage_parameters, result)
+        # re-uses transaction with serializable
+        # Disabling vacuum to avoid any issues during the process
+        result = Query.storage_parameters_for(client, client.table, true) || ""
+        Store.set(:primary_table_storage_parameters, result)
 
         logger.debug("Disabling vacuum on shadow and audit table",
                      { shadow_table: shadow_table, audit_table: audit_table })
@@ -156,32 +176,38 @@ module PgOnlineSchemaChange
             autovacuum_enabled = false, toast.autovacuum_enabled = false
           );
         SQL
-        Query.run(client.connection, sql)
+        Query.run(client.connection, sql, true)
       end
 
       def run_alter_statement!
+        # re-uses transaction with serializable
         statement = Query.alter_statement_for(client, shadow_table)
         logger.info("Running alter statement on shadow table",
                     { shadow_table: shadow_table, parent_table: client.table })
-        Query.run(client.connection, statement)
+        Query.run(client.connection, statement, true)
 
         Store.set(:dropped_columns_list, Query.dropped_columns(client))
         Store.set(:renamed_columns_list, Query.renamed_columns(client))
       end
 
-      # Begin the process to copy data into copy table
-      # depending on the size of the table, this can be a time
-      # taking operation.
       def copy_data!
-        logger.info("Copying contents..", { shadow_table: shadow_table, parent_table: client.table })
+        # re-uses transaction with serializable
+        # Begin the process to copy data into copy table
+        # depending on the size of the table, this can be a time
+        # taking operation.
+        logger.info("Clearing contents of audit table before copy..", { shadow_table: shadow_table, parent_table: client.table })
+        Query.run(client.connection, "DELETE FROM #{audit_table}", true)
 
+        logger.info("Copying contents..", { shadow_table: shadow_table, parent_table: client.table })
         if client.copy_statement
           query = format(client.copy_statement, shadow_table: shadow_table)
-          return Query.run(client.connection, query)
+          return Query.run(client.connection, query, true)
         end
 
         sql = Query.copy_data_statement(client, shadow_table)
-        Query.run(client.connection, sql)
+        Query.run(client.connection, sql, true)
+      ensure
+        Query.run(client.connection, "COMMIT;") # commit the serializable transaction
       end
 
       def replay_and_swap!
@@ -255,8 +281,10 @@ module PgOnlineSchemaChange
         Query.run(client.connection, sql)
       end
 
-      private def random_string
-        @random_string ||= SecureRandom.hex(3)
+      private
+
+      def pgosc_identifier
+        @pgosc_identifier ||= SecureRandom.hex(3)
       end
     end
   end

data/lib/pg_online_schema_change/query.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "pg_query"
 require "pg"
 
@@ -5,7 +7,7 @@ module PgOnlineSchemaChange
   class Query
     extend Helper
 
-    INDEX_SUFFIX = "_pgosc".freeze
+    INDEX_SUFFIX = "_pgosc"
     DROPPED_COLUMN_TYPE = :AT_DropColumn
     RENAMED_COLUMN_TYPE = :AT_RenameColumn
     LOCK_ATTEMPT = 4
@@ -15,28 +17,28 @@ module PgOnlineSchemaChange
         PgQuery.parse(query).tree.stmts.all? do |statement|
           statement.stmt.alter_table_stmt.instance_of?(PgQuery::AlterTableStmt) || statement.stmt.rename_stmt.instance_of?(PgQuery::RenameStmt)
         end
-      rescue PgQuery::ParseError => e
+      rescue PgQuery::ParseError
         false
       end
 
       def same_table?(query)
-        tables = PgQuery.parse(query).tree.stmts.map do |statement|
+        tables = PgQuery.parse(query).tree.stmts.filter_map do |statement|
           if statement.stmt.alter_table_stmt.instance_of?(PgQuery::AlterTableStmt)
             statement.stmt.alter_table_stmt.relation.relname
           elsif statement.stmt.rename_stmt.instance_of?(PgQuery::RenameStmt)
             statement.stmt.rename_stmt.relation.relname
           end
-        end.compact
+        end
 
         tables.uniq.count == 1
-      rescue PgQuery::ParseError => e
+      rescue PgQuery::ParseError
         false
       end
 
       def table(query)
-        from_rename_statement = PgQuery.parse(query).tree.stmts.map do |statement|
+        from_rename_statement = PgQuery.parse(query).tree.stmts.filter_map do |statement|
           statement.stmt.rename_stmt&.relation&.relname
-        end.compact[0]
+        end[0]
         PgQuery.parse(query).tables[0] || from_rename_statement
       end
 
@@ -48,7 +50,7 @@ module PgOnlineSchemaChange
         connection.async_exec("BEGIN;")
 
         result = connection.async_exec(query, &block)
-      rescue Exception
+      rescue Exception # rubocop:disable Lint/RescueException
         connection.cancel if connection.transaction_status != PG::PQTRANS_IDLE
         connection.block
         logger.info("Exception raised, rolling back query", { rollback: true, query: query })
@@ -60,7 +62,7 @@ module PgOnlineSchemaChange
         result
       end
 
-      def table_columns(client, table = nil)
+      def table_columns(client, table = nil, reuse_trasaction = false)
         sql = <<~SQL
           SELECT attname as column_name, format_type(atttypid, atttypmod) as type, attnum as column_position FROM   pg_attribute
           WHERE  attrelid = \'#{table || client.table}\'::regclass AND attnum > 0 AND NOT attisdropped
@@ -68,7 +70,7 @@ module PgOnlineSchemaChange
         SQL
         mapped_columns = []
 
-        run(client.connection, sql) do |result|
+        run(client.connection, sql, reuse_trasaction) do |result|
           mapped_columns = result.map do |row|
             row["column_name_regular"] = row["column_name"]
             row["column_name"] = client.connection.quote_ident(row["column_name"])
@@ -144,11 +146,11 @@ module PgOnlineSchemaChange
         end
 
         references.map do |row|
-          if row["definition"].end_with?("NOT VALID")
-            add_statement = "ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]};"
-          else
-            add_statement = "ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]} NOT VALID;"
-          end
+          add_statement = if row["definition"].end_with?("NOT VALID")
+                            "ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]};"
+                          else
+                            "ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]} NOT VALID;"
+                          end
 
           drop_statement = "ALTER TABLE #{row["table_on"]} DROP CONSTRAINT #{row["constraint_name"]};"
 
@@ -210,13 +212,13 @@ module PgOnlineSchemaChange
         columns.first
       end
 
-      def storage_parameters_for(client, table)
+      def storage_parameters_for(client, table, reuse_trasaction = false)
         query = <<~SQL
           SELECT array_to_string(reloptions, ',') as params FROM pg_class WHERE relname=\'#{table}\';
         SQL
 
         columns = []
-        run(client.connection, query) do |result|
+        run(client.connection, query, reuse_trasaction) do |result|
           columns = result.map { |row| row["params"] }
         end
 
@@ -266,8 +268,8 @@ module PgOnlineSchemaChange
         run(client.connection, query, true)
       end
 
-      def copy_data_statement(client, shadow_table)
-        select_columns = table_columns(client).map do |entry|
+      def copy_data_statement(client, shadow_table, reuse_trasaction = false)
+        select_columns = table_columns(client, client.table, reuse_trasaction).map do |entry|
           entry["column_name_regular"]
         end
 
@@ -291,7 +293,7 @@ module PgOnlineSchemaChange
           client.connection.quote_ident(select_column)
         end
 
-        sql = <<~SQL
+        <<~SQL
           INSERT INTO #{shadow_table}(#{insert_into_columns.join(", ")})
           SELECT #{select_columns.join(", ")}
           FROM ONLY #{client.table}

data/lib/pg_online_schema_change/replay.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
+# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+
 module PgOnlineSchemaChange
   class Replay
     extend Helper
@@ -5,7 +9,6 @@ module PgOnlineSchemaChange
     class << self
       PULL_BATCH_COUNT = 1000
       DELTA_COUNT = 20
-      RESERVED_COLUMNS = %w[operation_type trigger_time].freeze
 
       # This, picks PULL_BATCH_COUNT rows by primary key from audit_table,
       # replays it on the shadow_table. Once the batch is done,
@@ -25,7 +28,7 @@ module PgOnlineSchemaChange
 
       def rows_to_play(reuse_trasaction = false)
         select_query = <<~SQL
-          SELECT * FROM #{audit_table} ORDER BY #{primary_key} LIMIT #{PULL_BATCH_COUNT};
+          SELECT * FROM #{audit_table} ORDER BY #{audit_table_pk} LIMIT #{PULL_BATCH_COUNT};
         SQL
 
         rows = []
@@ -34,6 +37,10 @@ module PgOnlineSchemaChange
         rows
       end
 
+      def reserved_columns
+        @reserved_columns ||= [trigger_time_column, operation_type_column, audit_table_pk]
+      end
+
       def play!(rows, reuse_trasaction = false)
         logger.info("Replaying rows, count: #{rows.size}")
 
@@ -44,7 +51,7 @@ module PgOnlineSchemaChange
 
           # Remove audit table cols, since we will be
           # re-mapping them for inserts and updates
-          RESERVED_COLUMNS.each do |col|
+          reserved_columns.each do |col|
             new_row.delete(col)
           end
 
@@ -73,7 +80,7 @@ module PgOnlineSchemaChange
             client.connection.escape_string(value)
           end
 
-          case row["operation_type"]
+          case row[operation_type_column]
           when "INSERT"
             values = new_row.map { |_, val| "'#{val}'" }.join(",")
 
@@ -110,13 +117,15 @@ module PgOnlineSchemaChange
         Query.run(client.connection, to_be_replayed.join, reuse_trasaction)
 
         # Delete items from the audit now that are replayed
-        if to_be_deleted_rows.count >= 1
-          delete_query = <<~SQL
-            DELETE FROM #{audit_table} WHERE #{primary_key} IN (#{to_be_deleted_rows.join(",")})
-          SQL
-          Query.run(client.connection, delete_query, reuse_trasaction)
-        end
+        return unless to_be_deleted_rows.count >= 1
+
+        delete_query = <<~SQL
+          DELETE FROM #{audit_table} WHERE #{primary_key} IN (#{to_be_deleted_rows.join(",")})
+        SQL
+        Query.run(client.connection, delete_query, reuse_trasaction)
       end
     end
   end
 end
+
+# rubocop:enable Metrics/AbcSize, Metrics/MethodLength

data/lib/pg_online_schema_change/store.rb

@@ -1,17 +1,21 @@
+# frozen_string_literal: true
+
 require "pg_query"
 require "pg"
 
 module PgOnlineSchemaChange
   class Store
     class << self
-      @@object = {}
+      @object = {}
 
       def get(key)
-        @@object[key.to_s] || @@object[key.to_sym]
+        @object ||= {}
+        @object[key.to_s] || @object[key.to_sym]
       end
 
       def set(key, value)
-        @@object[key.to_sym] = value
+        @object ||= {}
+        @object[key.to_sym] = value
       end
     end
   end

data/lib/pg_online_schema_change/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgOnlineSchemaChange
-  VERSION = "0.1.0"
+  VERSION = "0.4.0"
 end

data/lib/pg_online_schema_change.rb

@@ -18,16 +18,12 @@ module PgOnlineSchemaChange
   class CountBelowDelta < StandardError; end
   class AccessExclusiveLockNotAcquired < StandardError; end
 
-  def self.logger=(verbose)
-    @@logger ||= begin
-        logger = Ougai::Logger.new($stdout)
-        logger.level = verbose ? Ougai::Logger::TRACE : Ougai::Logger::INFO
-        logger.with_fields = { version: PgOnlineSchemaChange::VERSION }
-        logger
-      end
-  end
-
-  def self.logger
-    @@logger
+  def self.logger(verbose: false)
+    @logger ||= begin
+      logger = Ougai::Logger.new($stdout)
+      logger.level = verbose ? Ougai::Logger::TRACE : Ougai::Logger::INFO
+      logger.with_fields = { version: PgOnlineSchemaChange::VERSION }
+      logger
+    end
   end
 end

data/scripts/release.sh

@@ -0,0 +1,28 @@
+export VERSION=$1
+echo "VERSION: ${VERSION}"
+
+echo "=== Pushing tags to github ===="
+git tag v$VERSION
+git push origin --tags
+
+echo "=== Building Gem ===="
+gem build pg_online_schema_change.gemspec
+
+echo "=== Pushing gem ===="
+gem push pg_online_schema_change-$VERSION.gem
+
+echo "=== Sleeping for 5s ===="
+sleep 5 
+
+echo "=== Building Image ===="
+docker build . --build-arg VERSION=$VERSION -t shayonj/pg-osc:$VERSION
+
+echo "=== Tagging Image ===="
+docker image tag shayonj/pg-osc:$VERSION shayonj/pg-osc:latest
+
+echo "=== Pushing Image ===="
+docker push shayonj/pg-osc:$VERSION
+docker push shayonj/pg-osc:latest
+
+echo "=== Cleaning up ===="
+rm pg_online_schema_change-$VERSION.gem