pg_online_schema_change 0.5.0 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b14af6aa2b98ab8f1b2aab5ae0a8555e017f92531ce1214e50f2ac62ff354224
-  data.tar.gz: 66ba81f4e90f4dc612d863d043838fec8676e468924eeb5c39d21094d002e48a
+  metadata.gz: c49f895f91c72ef491002660c994088b9d527db88d3b5d77cbb851771a2f2ac8
+  data.tar.gz: d57329d32657cd64edb60b8c174c6fc32f4279a202548b5132407e98c59f2f9e
 SHA512:
-  metadata.gz: a272a3749f8f053a528d859cc35b97b5fde7d6353a3c0c710500e35d33df6b4bc8526b18e855fd154b742eef8189b661e8e710a6b25221df6e47d55a47381a2e
-  data.tar.gz: c0dd7f41e204840b54b768df1f1e8486f79038bede199021ef13684c4b1369e3de43be6fbfd8f03b7ca652251c59a8a580ecefb235e00e54858d77f88a7334de
+  metadata.gz: 963c4c9751301d029857f021b40d033d50f9b8865548a5d8bd752ed4c03a4cf1e5b016c5129e771db45df1e2677a8e29b5c2b488f152932238e7158e4c3a2a8b
+  data.tar.gz: bc8c559ade623ea76794cf1756642474cf32eee08406f884503b9450e1d5b39233778dd77b7fcd56327c140ba046279b21a1a1a64b5ac606bf5c2119cea6b350

data/.rspec

@@ -1,4 +1,5 @@
 --format documentation
 --color
 --require spec_helper
---fail-fast
+--fail-fast
+--exclude-pattern spec/lib/smoke_spec.rb

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2022-02-21 22:46:44 UTC using RuboCop version 1.23.0.
+# on 2022-03-13 19:35:49 UTC using RuboCop version 1.23.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -9,7 +9,7 @@
 # Offense count: 2
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
-  Max: 233
+  Max: 250
 
 # Offense count: 2
 # Configuration parameters: IgnoredMethods.
@@ -26,14 +26,14 @@ Packaging/GemspecGit:
   Exclude:
     - 'pg_online_schema_change.gemspec'
 
-# Offense count: 62
+# Offense count: 67
 # Configuration parameters: CountAsOne.
 RSpec/ExampleLength:
   Max: 55
 
-# Offense count: 38
+# Offense count: 24
 RSpec/MultipleExpectations:
-  Max: 14
+  Max: 13
 
 # Offense count: 6
 # Configuration parameters: AllowedMethods.

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+
+## [0.6.0] - 2022-03-13
+* Move CI to Github Actions in https://github.com/shayonj/pg-osc/pull/64
+* Set --password as optional since it is deprecated in https://github.com/shayonj/pg-osc/pull/66
+* Smoke tests in https://github.com/shayonj/pg-osc/pull/65
+* Add foreign keys to parent during swap in https://github.com/shayonj/pg-osc/pull/67
+
+## [0.6.0] - 2022-02-26
+* Delete items by audit table PK when replaying by @shayonj @jfrost in https://github.com/shayonj/pg-osc/pull/60
+  - Fixes a race condition issue: https://github.com/shayonj/pg-osc/issues/58
+
+## [0.5.0] - 2022-02-26
+* Share some preliminary load test figures in https://github.com/shayonj/pg-osc/pull/54
+* Reuse existing transaction open for reading table columns in https://github.com/shayonj/pg-osc/pull/53
+* Start to deprecate --password with PGPASSWORD in https://github.com/shayonj/pg-osc/pull/56
+* Introduce configurable PULL_BATCH_COUNT and DELTA_COUNT in https://github.com/shayonj/pg-osc/pull/57
+
 ## [0.4.0] - 2022-02-22
 * Lint sourcecode, setup Rubocop proper and Lint in CI by @shayonj in https://github.com/shayonj/pg-osc/pull/46
 * Uniquely identify operation_type column by @shayonj in https://github.com/shayonj/pg-osc/pull/50

data/CODE_OF_CONDUCT.md

@@ -39,7 +39,7 @@ This Code of Conduct applies within all community spaces, and also applies when
 
 ## Enforcement
 
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at shayon@loom.com. All complaints will be reviewed and investigated promptly and fairly.
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at shayonj@gmail.com. All complaints will be reviewed and investigated promptly and fairly.
 
 All community leaders are obligated to respect the privacy and security of the reporter of any incident.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pg_online_schema_change (0.4.0)
+    pg_online_schema_change (0.7.0)
       ougai (~> 2.0.0)
       pg (~> 1.3.2)
       pg_query (~> 2.1.3)
@@ -14,6 +14,7 @@ GEM
     coderay (1.1.3)
     diff-lcs (1.5.0)
     google-protobuf (3.19.4)
+    google-protobuf (3.19.4-x86_64-linux)
     method_source (1.0.0)
     oj (3.13.11)
     ougai (2.0.0)
@@ -21,7 +22,7 @@ GEM
     parallel (1.21.0)
     parser (3.0.3.2)
       ast (~> 2.4.1)
-    pg (1.3.2)
+    pg (1.3.4)
     pg_query (2.1.3)
       google-protobuf (>= 3.19.2)
     pry (0.14.1)

data/README.md

@@ -1,6 +1,8 @@
 # pg-osc
 
-[![CircleCI](https://circleci.com/gh/shayonj/pg-osc/tree/main.svg?style=shield)](https://circleci.com/gh/shayonj/pg-osc/tree/main)
+[![CI](https://github.com/shayonj/pg-osc/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg-osc/actions/workflows/ci.yaml)
+[![Smoke Test PG 9.6](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-9-6.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-9-6.yaml)
+[![Smoke Test PG 13.6](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-13-6.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-13-6.yaml)
 [![Gem Version](https://badge.fury.io/rb/pg_online_schema_change.svg)](https://badge.fury.io/rb/pg_online_schema_change)
 
 pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALTER` statements) in Postgres tables with minimal locks, thus helping achieve zero downtime schema changes against production workloads. 
@@ -100,6 +102,7 @@ print the version
 - `pg-osc` acquires minimal locks throughout the process (read more below on the caveats).
 - Copies over indexes and Foreign keys.
 - Optionally drop or retain old tables in the end.
+- Tune how slow or fast should replays be from the audit/log table ([Replaying larger workloads](#replaying-larger-workloads)).
 - Backfill old/new columns as data is copied from primary table to shadow table, and then perform the swap. [Example](#backfill-data)
 - **TBD**: Ability to reverse the change with no data loss. [tracking issue](https://github.com/shayonj/pg-osc/issues/14)
 

data/lib/pg_online_schema_change/cli.rb

@@ -15,7 +15,7 @@ module PgOnlineSchemaChange
     method_option :host, aliases: "-h", type: :string, required: true, desc: "Server host where the Database is located"
     method_option :username, aliases: "-u", type: :string, required: true, desc: "Username for the Database"
     method_option :port, aliases: "-p", type: :numeric, required: true, default: 5432, desc: "Port for the Database"
-    method_option :password, aliases: "-w", type: :string, required: true, desc: "DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead."
+    method_option :password, aliases: "-w", type: :string, required: false, default: "", desc: "DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead."
     method_option :verbose, aliases: "-v", type: :boolean, default: false, desc: "Emit logs in debug mode"
     method_option :drop, aliases: "-f", type: :boolean, default: false,
                          desc: "Drop the original table in the end after the swap"

data/lib/pg_online_schema_change/functions.rb

@@ -47,18 +47,6 @@ FUNC_CREATE_TABLE_ALL = <<~SQL
     EXECUTE format(
           'CREATE TABLE %s (LIKE %s including all)',
           newsource_table, source_table);
-      for rec in
-          SELECT oid, conname
-          FROM pg_constraint
-          WHERE contype = 'f'
-          AND conrelid = source_table::regclass
-      LOOP
-      EXECUTE format(
-              'ALTER TABLE %s add constraint %s %s',
-              newsource_table,
-              rec.conname,
-              pg_get_constraintdef(rec.oid));
-      END LOOP;
     END
   $$;
 SQL

data/lib/pg_online_schema_change/orchestrate.rb

@@ -37,6 +37,9 @@ module PgOnlineSchemaChange
         Store.set(:audit_table_pk, "at_#{pgosc_identifier}_id")
         Store.set(:audit_table_pk_sequence, "#{audit_table}_#{audit_table_pk}_seq")
         Store.set(:shadow_table, "pgosc_st_#{client.table}_#{pgosc_identifier}")
+
+        Store.set(:referential_foreign_key_statements, Query.referential_foreign_keys_to_refresh(client, client.table))
+        Store.set(:self_foreign_key_statements, Query.self_foreign_keys_to_refresh(client, client.table))
       end
 
       def run!(options)
@@ -221,7 +224,6 @@ module PgOnlineSchemaChange
       def swap!
         logger.info("Performing swap!")
 
-        foreign_key_statements = Query.get_foreign_keys_to_refresh(client, client.table)
         storage_params_reset = primary_table_storage_parameters.empty? ? "" : "ALTER TABLE #{client.table} SET (#{primary_table_storage_parameters});"
 
         # From here on, all statements are carried out in a single
@@ -239,12 +241,13 @@ module PgOnlineSchemaChange
         sql = <<~SQL
           ALTER TABLE #{client.table} RENAME to #{old_primary_table};
           ALTER TABLE #{shadow_table} RENAME to #{client.table};
-          #{foreign_key_statements}
+          #{referential_foreign_key_statements}
+          #{self_foreign_key_statements}
           #{storage_params_reset}
           DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table};
         SQL
 
-        Query.run(client.connection, sql)
+        Query.run(client.connection, sql, opened)
       ensure
         Query.run(client.connection, "COMMIT;")
         Query.run(client.connection, "SET statement_timeout = 0;")
@@ -270,6 +273,7 @@ module PgOnlineSchemaChange
         shadow_table_drop = shadow_table ? "DROP TABLE IF EXISTS #{shadow_table}" : ""
 
         sql = <<~SQL
+          DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table};
           #{audit_table_drop};
           #{shadow_table_drop};
           #{primary_drop}

data/lib/pg_online_schema_change/query.rb

@@ -140,7 +140,7 @@ module PgOnlineSchemaChange
         end
       end
 
-      def get_foreign_keys_to_refresh(client, table)
+      def referential_foreign_keys_to_refresh(client, table)
         references = get_all_constraints_for(client).select do |row|
           row["table_from"] == table && row["constraint_type"] == "f"
         end
@@ -158,12 +158,32 @@ module PgOnlineSchemaChange
         end.join
       end
 
-      def get_foreign_keys_to_validate(client, table)
+      def self_foreign_keys_to_refresh(client, table)
         references = get_all_constraints_for(client).select do |row|
-          row["table_from"] == table && row["constraint_type"] == "f"
+          row["table_on"] == table && row["constraint_type"] == "f"
         end
 
         references.map do |row|
+          add_statement = if row["definition"].end_with?("NOT VALID")
+                            "ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]};"
+                          else
+                            "ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]} NOT VALID;"
+                          end
+          add_statement
+        end.join
+      end
+
+      def get_foreign_keys_to_validate(client, table)
+        constraints = get_all_constraints_for(client)
+        referential_foreign_keys = constraints.select do |row|
+          row["table_from"] == table && row["constraint_type"] == "f"
+        end
+
+        self_foreign_keys = constraints.select do |row|
+          row["table_on"] == table && row["constraint_type"] == "f"
+        end
+
+        [referential_foreign_keys, self_foreign_keys].flatten.map do |row|
           "ALTER TABLE #{row["table_on"]} VALIDATE CONSTRAINT #{row["constraint_name"]};"
         end.join
       end

data/lib/pg_online_schema_change/replay.rb

@@ -87,7 +87,7 @@ module PgOnlineSchemaChange
             SQL
             to_be_replayed << sql
 
-            to_be_deleted_rows << "'#{row[primary_key]}'"
+            to_be_deleted_rows << "'#{row[audit_table_pk]}'"
           when "UPDATE"
             set_values = new_row.map do |column, value|
               "#{column} = '#{value}'"
@@ -100,14 +100,14 @@ module PgOnlineSchemaChange
             SQL
             to_be_replayed << sql
 
-            to_be_deleted_rows << "'#{row[primary_key]}'"
+            to_be_deleted_rows << "'#{row[audit_table_pk]}'"
           when "DELETE"
             sql = <<~SQL
               DELETE FROM #{shadow_table} WHERE #{primary_key}=\'#{row[primary_key]}\';
             SQL
             to_be_replayed << sql
 
-            to_be_deleted_rows << "'#{row[primary_key]}'"
+            to_be_deleted_rows << "'#{row[audit_table_pk]}'"
           end
         end
 
@@ -117,7 +117,7 @@ module PgOnlineSchemaChange
         return unless to_be_deleted_rows.count >= 1
 
         delete_query = <<~SQL
-          DELETE FROM #{audit_table} WHERE #{primary_key} IN (#{to_be_deleted_rows.join(",")})
+          DELETE FROM #{audit_table} WHERE #{audit_table_pk} IN (#{to_be_deleted_rows.join(",")})
         SQL
         Query.run(client.connection, delete_query, reuse_trasaction)
       end

data/lib/pg_online_schema_change/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgOnlineSchemaChange
-  VERSION = "0.5.0"
+  VERSION = "0.7.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_online_schema_change
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-26 00:00:00.000000000 Z
+date: 2022-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai