pg_online_schema_change 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3bc78403dc036598371393d5be0914bd7150660eda9798fb827af2892d058396
-  data.tar.gz: 6dbfb6e41267accf02e0da8ba88d7ea5039aa368ddc497e40d81f8952bed811d
+  metadata.gz: 273557a58c9492e628277e6eed7b655233c47bb51b12a61ca3eaf3377557205a
+  data.tar.gz: 6ec45511249401c420cc3c61652a4ad18225917b644574fb58329819fb422ab1
 SHA512:
-  metadata.gz: '0683dd33e681162a1b16b471fbefc52021b66ccbb3235994fd0654c6d97052a5482e22e29c1280cbc332eb8510b1b4a571075b4e36f9a2565c359ad086100d05'
-  data.tar.gz: 51e1fcd9d8d8d4f2a3f5760dee1e51fbde5a45f3bf7dd9c2d048390856b3b71bfb23333b053e0a0e43c026e90a9f704efbc903760cd37e95eb5ac5ea24b66512
+  metadata.gz: 29f07b0af4c6a78de6945b6f43e271518f9c5953388e99dcc7582c4db865881ac5e5b8d0f1dff9a1215216ea534ad1e393fb268816e3799832911688b7abc14e
+  data.tar.gz: 0362f9c666c58acdb2fb63f1cd0208c719eb788555b218a57e2aee7bb9d98bbd9a7ff4d1e0ee0a56deec8e9679def1ae116fef919366e86ae30b2444d3379e6a

data/.rspec

@@ -1,4 +1,5 @@
 --format documentation
 --color
 --require spec_helper
---fail-fast
+--fail-fast
+--exclude-pattern spec/lib/smoke_spec.rb

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2022-02-21 22:46:44 UTC using RuboCop version 1.23.0.
+# on 2022-03-13 19:35:49 UTC using RuboCop version 1.23.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -9,7 +9,7 @@
 # Offense count: 2
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
-  Max: 233
+  Max: 250
 
 # Offense count: 2
 # Configuration parameters: IgnoredMethods.
@@ -26,14 +26,14 @@ Packaging/GemspecGit:
   Exclude:
     - 'pg_online_schema_change.gemspec'
 
-# Offense count: 62
+# Offense count: 67
 # Configuration parameters: CountAsOne.
 RSpec/ExampleLength:
   Max: 55
 
-# Offense count: 38
+# Offense count: 24
 RSpec/MultipleExpectations:
-  Max: 14
+  Max: 13
 
 # Offense count: 6
 # Configuration parameters: AllowedMethods.

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## [0.6.0] - 2022-02-26
+* Delete items by audit table PK when replaying by @shayonj @jfrost in https://github.com/shayonj/pg-osc/pull/60
+  - Fixes a race condition issue: https://github.com/shayonj/pg-osc/issues/58
+
 ## [0.5.0] - 2022-02-26
 * Share some preliminary load test figures in https://github.com/shayonj/pg-osc/pull/54
 * Reuse existing transaction open for reading table columns in https://github.com/shayonj/pg-osc/pull/53

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pg_online_schema_change (0.5.0)
+    pg_online_schema_change (0.6.0)
       ougai (~> 2.0.0)
       pg (~> 1.3.2)
       pg_query (~> 2.1.3)
@@ -14,6 +14,7 @@ GEM
     coderay (1.1.3)
     diff-lcs (1.5.0)
     google-protobuf (3.19.4)
+    google-protobuf (3.19.4-x86_64-linux)
     method_source (1.0.0)
     oj (3.13.11)
     ougai (2.0.0)
@@ -21,7 +22,7 @@ GEM
     parallel (1.21.0)
     parser (3.0.3.2)
       ast (~> 2.4.1)
-    pg (1.3.2)
+    pg (1.3.3)
     pg_query (2.1.3)
       google-protobuf (>= 3.19.2)
     pry (0.14.1)

data/README.md

@@ -1,6 +1,8 @@
 # pg-osc
 
-[![CircleCI](https://circleci.com/gh/shayonj/pg-osc/tree/main.svg?style=shield)](https://circleci.com/gh/shayonj/pg-osc/tree/main)
+[![CI](https://github.com/shayonj/pg-osc/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg-osc/actions/workflows/ci.yaml)
+[![Smoke Test PG 9.6](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-9-6.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-9-6.yaml)
+[![Smoke Test PG 13.6](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-13-6.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-13-6.yaml)
 [![Gem Version](https://badge.fury.io/rb/pg_online_schema_change.svg)](https://badge.fury.io/rb/pg_online_schema_change)
 
 pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALTER` statements) in Postgres tables with minimal locks, thus helping achieve zero downtime schema changes against production workloads. 

data/lib/pg_online_schema_change/cli.rb

@@ -15,7 +15,7 @@ module PgOnlineSchemaChange
     method_option :host, aliases: "-h", type: :string, required: true, desc: "Server host where the Database is located"
     method_option :username, aliases: "-u", type: :string, required: true, desc: "Username for the Database"
     method_option :port, aliases: "-p", type: :numeric, required: true, default: 5432, desc: "Port for the Database"
-    method_option :password, aliases: "-w", type: :string, required: true, desc: "DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead."
+    method_option :password, aliases: "-w", type: :string, required: false, default: "", desc: "DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead."
     method_option :verbose, aliases: "-v", type: :boolean, default: false, desc: "Emit logs in debug mode"
     method_option :drop, aliases: "-f", type: :boolean, default: false,
                          desc: "Drop the original table in the end after the swap"

data/lib/pg_online_schema_change/functions.rb

@@ -47,18 +47,6 @@ FUNC_CREATE_TABLE_ALL = <<~SQL
     EXECUTE format(
           'CREATE TABLE %s (LIKE %s including all)',
           newsource_table, source_table);
-      for rec in
-          SELECT oid, conname
-          FROM pg_constraint
-          WHERE contype = 'f'
-          AND conrelid = source_table::regclass
-      LOOP
-      EXECUTE format(
-              'ALTER TABLE %s add constraint %s %s',
-              newsource_table,
-              rec.conname,
-              pg_get_constraintdef(rec.oid));
-      END LOOP;
     END
   $$;
 SQL

data/lib/pg_online_schema_change/orchestrate.rb

@@ -38,7 +38,8 @@ module PgOnlineSchemaChange
         Store.set(:audit_table_pk_sequence, "#{audit_table}_#{audit_table_pk}_seq")
         Store.set(:shadow_table, "pgosc_st_#{client.table}_#{pgosc_identifier}")
 
-        Store.set(:foreign_key_statements, Query.get_foreign_keys_to_refresh(client, client.table))
+        Store.set(:referential_foreign_key_statements, Query.referential_foreign_keys_to_refresh(client, client.table))
+        Store.set(:self_foreign_key_statements, Query.self_foreign_keys_to_refresh(client, client.table))
       end
 
       def run!(options)
@@ -240,7 +241,8 @@ module PgOnlineSchemaChange
         sql = <<~SQL
           ALTER TABLE #{client.table} RENAME to #{old_primary_table};
           ALTER TABLE #{shadow_table} RENAME to #{client.table};
-          #{foreign_key_statements}
+          #{referential_foreign_key_statements}
+          #{self_foreign_key_statements}
           #{storage_params_reset}
           DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table};
         SQL

data/lib/pg_online_schema_change/query.rb

@@ -140,7 +140,7 @@ module PgOnlineSchemaChange
         end
       end
 
-      def get_foreign_keys_to_refresh(client, table)
+      def referential_foreign_keys_to_refresh(client, table)
         references = get_all_constraints_for(client).select do |row|
           row["table_from"] == table && row["constraint_type"] == "f"
         end
@@ -158,12 +158,32 @@ module PgOnlineSchemaChange
         end.join
       end
 
-      def get_foreign_keys_to_validate(client, table)
+      def self_foreign_keys_to_refresh(client, table)
         references = get_all_constraints_for(client).select do |row|
-          row["table_from"] == table && row["constraint_type"] == "f"
+          row["table_on"] == table && row["constraint_type"] == "f"
         end
 
         references.map do |row|
+          add_statement = if row["definition"].end_with?("NOT VALID")
+                            "ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]};"
+                          else
+                            "ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]} NOT VALID;"
+                          end
+          add_statement
+        end.join
+      end
+
+      def get_foreign_keys_to_validate(client, table)
+        constraints = get_all_constraints_for(client)
+        referential_foreign_keys = constraints.select do |row|
+          row["table_from"] == table && row["constraint_type"] == "f"
+        end
+
+        self_foreign_keys = constraints.select do |row|
+          row["table_on"] == table && row["constraint_type"] == "f"
+        end
+
+        [referential_foreign_keys, self_foreign_keys].flatten.map do |row|
           "ALTER TABLE #{row["table_on"]} VALIDATE CONSTRAINT #{row["constraint_name"]};"
         end.join
       end

data/lib/pg_online_schema_change/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgOnlineSchemaChange
-  VERSION = "0.6.0"
+  VERSION = "0.7.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_online_schema_change
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-27 00:00:00.000000000 Z
+date: 2022-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai