pg_easy_replicate 0.2.7 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90a3489ff7f33639118b99b142c2dcdef491ad384df80de0cceee89b706a9073
-  data.tar.gz: 0ab288970a9eb6465878ef03aca0d67f7543426a4cc744a4e47592ac200f5b68
+  metadata.gz: 846d76c9da382a82b36243070402e2e42d7135c6ccd52a1308b4d4ebdb15e4ca
+  data.tar.gz: 159925fe9649bc527a861f4e5eac52102af05542ddfeb8f742e9077a1764fe38
 SHA512:
-  metadata.gz: e487ac3b147855379c230d3b792d702d42776eb254f227cf31a8ebc5dee677cbf47b2efd7a9c0c218a87f064b1449e3c286d1dbc6c0f50f6b108cee3b1a5ccaa
-  data.tar.gz: 5ce88fff42f5ba68977b15261376d48e23173d210eddd1beccdad148cafab45f8e760a09d37991cef491313b4dee12dfa761b495fa1b90e8348c2b89ed2c3a6a
+  metadata.gz: 512c5ec0e049476d5e02a549effd6d9032642e59eda2507a6a611fae2dfcb136c1efb8fab58f1dd46b12a38dd758195792424415b97e445ad21cb28737048a2c
+  data.tar.gz: c36d0cfaebf53e89b58bbdcd52f8394d67196f912bcaaeb2247122bbb1a431d7b850d6c1355058cb810ea718ec9cb268204b568c36aeca98b90174ed67544714

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    pg_easy_replicate (0.2.7)
+    pg_easy_replicate (0.3.0)
       ougai (~> 2.0.0)
       pg (~> 1.5.3)
-      sequel (>= 5.69, < 5.83)
+      pg_query (~> 5.1.0)
+      sequel (>= 5.69, < 5.84)
       thor (>= 1.2.2, < 1.4.0)
 
 GEM
@@ -14,6 +15,12 @@ GEM
     bigdecimal (3.1.8)
     coderay (1.1.3)
     diff-lcs (1.5.1)
+    google-protobuf (4.28.0-arm64-darwin)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.28.0-x86_64-linux)
+      bigdecimal
+      rake (>= 13)
     haml (6.1.1)
       temple (>= 0.8.2)
       thor
@@ -29,6 +36,8 @@ GEM
       ast (~> 2.4.1)
       racc
     pg (1.5.7)
+    pg_query (5.1.0)
+      google-protobuf (>= 3.22.3)
     prettier_print (1.2.1)
     pry (0.14.2)
       coderay (~> 1.1)
@@ -38,7 +47,7 @@ GEM
     rake (13.2.1)
     rbs (3.1.0)
     regexp_parser (2.9.2)
-    rexml (3.3.3)
+    rexml (3.3.6)
       strscan
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -85,7 +94,7 @@ GEM
     rubocop-rspec_rails (2.28.2)
       rubocop (~> 1.40)
     ruby-progressbar (1.13.0)
-    sequel (5.82.0)
+    sequel (5.83.1)
       bigdecimal
     strscan (3.1.0)
     syntax_tree (6.2.0)

data/README.md

@@ -22,9 +22,14 @@ Battle tested in production at [Tines](https://www.tines.com/) 🚀
     - [Config Check](#config-check-1)
     - [Bootstrap](#bootstrap-1)
   - [Start sync](#start-sync)
+  - [DDL Changes Management](#ddl-changes-management)
+    - [Listing DDL Changes](#listing-ddl-changes)
+    - [Applying DDL Changes](#applying-ddl-changes)
   - [Stats](#stats)
   - [Performing switchover](#performing-switchover)
 - [Replicating single database with custom tables](#replicating-single-database-with-custom-tables)
+- [Exclude tables from replication](#exclude-tables-from-replication)
+  - [Cleanup](#cleanup)
 - [Switchover strategies with minimal downtime](#switchover-strategies-with-minimal-downtime)
   - [Rolling restart strategy](#rolling-restart-strategy)
   - [DNS Failover strategy](#dns-failover-strategy)
@@ -69,11 +74,13 @@ All [Logical Replication Restrictions](https://www.postgresql.org/docs/current/l
 
 ## Usage
 
-Ensure `SOURCE_DB_URL` and `TARGET_DB_URL` are present as environment variables in the runtime environment. 
+Ensure `SOURCE_DB_URL` and `TARGET_DB_URL` are present as environment variables in the runtime environment.
+
 - `SOURCE_DB_URL` = The database that you want to replicate FROM.
 - `TARGET_DB_URL` = The database that you want to replicate TO.
 
 The URL should be in postgres connection string format. Example:
+
 ```bash
 $ export SOURCE_DB_URL="postgres://USERNAME:PASSWORD@localhost:5432/DATABASE_NAME"
 $ export TARGET_DB_URL="postgres://USERNAME:PASSWORD@localhost:5433/DATABASE_NAME"
@@ -174,12 +181,62 @@ Once the bootstrap is complete, you can start the sync. Starting the sync sets u
 **NOTE**: Start sync by default will drop all indices in the target database for performance reasons. And will automatically re-add the indices during `switchover`. It is turned on by default and you can opt out of this with `--no-recreate-indices-post-copy`
 
 ```bash
-$ pg_easy_replicate start_sync --group-name database-cluster-1
+$ pg_easy_replicate start_sync --group-name database-cluster-1 [-d <track-ddl>]
 
 {"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":22113,"level":30,"time":"2023-06-19T15:54:54.874-04:00","v":0,"msg":"Setting up publication","publication_name":"pger_publication_database_cluster_1","version":"0.1.0"}
 ...
 ```
 
+### DDL Changes Management
+
+`pg_easy_replicate` now supports tracking and applying DDL (Data Definition Language) changes between the source and target databases. To track DDLs you can pass `-track-ddl` to `start_sync`.
+
+This feature ensures that most schema changes made to the source database tables that are being replicated during the replication process are tracked, so that you can apply them at your will before or after switchover.
+
+#### Listing DDL Changes
+
+To view the DDL changes that have been tracked:
+
+```bash
+$ pg_easy_replicate list_ddl_changes -g <group-name> [-l <limit>]
+```
+
+This command will display a list of DDL changes in JSON format;
+
+```
+[
+  {
+    "id": 1,
+    "group_name": "cluster-1",
+    "event_type": "ddl_command_end",
+    "object_type": "table",
+    "object_identity": "public.pgbench_accounts",
+    "ddl_command": "ALTER TABLE public.pgbench_accounts ADD COLUMN test_column VARCHAR(255)",
+    "created_at": "2024-08-31 15:42:33 UTC"
+  }
+]
+```
+
+#### Applying DDL Changes
+
+`pg_easy_replicate` won't automatically apply the changes for you. To apply the tracked DDL changes to the target database:
+
+```bash
+$ pg_easy_replicate apply_ddl_change -g <group-name> [-i <change-id>]
+```
+
+If you specify a change ID with the `-i` option, only that specific change will be applied. If you don't specify an ID, you'll be prompted to apply all pending changes.
+
+```bash
+$ pg_easy_replicate apply_ddl_change -g cluster-1
+The following DDL changes will be applied:
+ID: 1, Type: table, Command: ALTER TABLE public.pgbench_accounts ADD COLUMN test_column VARCHAR(255)...
+
+Do you want to apply all these changes? (y/n): y
+...
+All pending DDL changes applied successfully.
+```
+
 ### Stats
 
 You can inspect or watch stats any time during the sync process. The stats give you an idea of when the sync started, current flush/write lag, how many tables are in `replicating`, `copying` or other stages, and more.

data/docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.7"
 services:
   source_db:
-    image: postgres:12
+    image: postgres:14
     ports:
       - "5432:5432"
     environment:
@@ -9,6 +9,7 @@ services:
       POSTGRES_PASSWORD: james-bond123@7!'3aaR
       POSTGRES_DB: postgres-db
     command: >
+      -c max_connections=200
       -c wal_level=logical
       -c ssl=on
       -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
@@ -25,6 +26,7 @@ services:
       POSTGRES_PASSWORD: james-bond123@7!'3aaR
       POSTGRES_DB: postgres-db
     command: >
+      -c max_connections=200
       -c wal_level=logical
       -c ssl=on
       -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem

data/lib/pg_easy_replicate/cli.rb

@@ -54,6 +54,11 @@ module PgEasyReplicate
                   aliases: "-c",
                   boolean: true,
                   desc: "Copy schema to the new database"
+    method_option :track_ddl,
+                  aliases: "-d",
+                  type: :boolean,
+                  default: false,
+                  desc: "Enable DDL tracking for the group"
     desc "bootstrap",
          "Sets up temporary tables for information required during runtime"
     def bootstrap
@@ -83,11 +88,6 @@ module PgEasyReplicate
                   aliases: "-g",
                   required: true,
                   desc: "Name of the group to provision"
-    method_option :group_name,
-                  aliases: "-g",
-                  required: true,
-                  desc:
-                    "Name of the grouping for this collection of source and target DB"
     method_option :schema_name,
                   aliases: "-s",
                   desc:
@@ -104,10 +104,14 @@ module PgEasyReplicate
                     "Comma separated list of table names to exclude. Default: None"
     method_option :recreate_indices_post_copy,
                   type: :boolean,
-                  default: true,
+                  default: false,
                   aliases: "-r",
                   desc:
                     "Drop all non-primary indices before copy and recreate them post-copy"
+    method_option :track_ddl,
+                  type: :boolean,
+                  default: false,
+                  desc: "Enable DDL tracking for the group"
     def start_sync
       PgEasyReplicate::Orchestrate.start_sync(options)
     end
@@ -122,7 +126,7 @@ module PgEasyReplicate
       PgEasyReplicate::Orchestrate.stop_sync(group_name: options[:group_name])
     end
 
-    desc "switchover ",
+    desc "switchover",
          "Puts the source database in read only mode after all the data is flushed and written"
     method_option :group_name,
                   aliases: "-g",
@@ -130,21 +134,18 @@ module PgEasyReplicate
                   desc: "Name of the group previously provisioned"
     method_option :lag_delta_size,
                   aliases: "-l",
-                  desc: "The size of the lag to watch for before switchover. Default 200KB."
+                  desc:
+                    "The size of the lag to watch for before switchover. Default 200KB."
     method_option :skip_vacuum_analyze,
                   type: :boolean,
                   default: false,
                   aliases: "-s",
                   desc: "Skip vacuum analyzing tables before switchover."
-    # method_option :bi_directional,
-    #               aliases: "-b",
-    #               desc:
-    #                 "Setup replication from target database to source database"
     def switchover
       PgEasyReplicate::Orchestrate.switchover(
         group_name: options[:group_name],
         lag_delta_size: options[:lag_delta_size],
-        skip_vacuum_analyze: options[:skip_vacuum_analyze]
+        skip_vacuum_analyze: options[:skip_vacuum_analyze],
       )
     end
 
@@ -162,6 +163,71 @@ module PgEasyReplicate
       end
     end
 
+    desc "list_ddl_changes", "Lists recent DDL changes in the source database"
+    method_option :group_name,
+                  aliases: "-g",
+                  required: true,
+                  desc: "Name of the group"
+    method_option :limit,
+                  aliases: "-l",
+                  type: :numeric,
+                  default: 100,
+                  desc: "Limit the number of DDL changes to display"
+    def list_ddl_changes
+      changes =
+        PgEasyReplicate::DDLManager.list_ddl_changes(
+          group_name: options[:group_name],
+          limit: options[:limit],
+        )
+      puts JSON.pretty_generate(changes)
+    end
+
+    desc "apply_ddl_change", "Applies DDL changes to the target database"
+    method_option :group_name,
+                  aliases: "-g",
+                  required: true,
+                  desc: "Name of the group"
+    method_option :id,
+                  aliases: "-i",
+                  type: :numeric,
+                  desc:
+                    "ID of the specific DDL change to apply. If not provided, all changes will be applied."
+    def apply_ddl_change
+      if options[:id]
+        PgEasyReplicate::DDLManager.apply_ddl_change(
+          group_name: options[:group_name],
+          id: options[:id],
+        )
+        puts "DDL change with ID #{options[:id]} applied successfully."
+      else
+        changes =
+          PgEasyReplicate::DDLManager.list_ddl_changes(
+            group_name: options[:group_name],
+          )
+        if changes.empty?
+          puts "No pending DDL changes to apply."
+          return
+        end
+
+        puts "The following DDL changes will be applied:"
+        changes.each do |change|
+          puts "ID: #{change[:id]}, Type: #{change[:object_type]}, Command: #{change[:ddl_command]}"
+        end
+        puts ""
+        print("Do you want to apply all these changes? (y/n): ")
+        confirmation = $stdin.gets.chomp.downcase
+
+        if confirmation == "y"
+          PgEasyReplicate::DDLManager.apply_all_ddl_changes(
+            group_name: options[:group_name],
+          )
+          puts "All pending DDL changes applied successfully."
+        else
+          puts "Operation cancelled."
+        end
+      end
+    end
+
     desc "version", "Prints the version"
     def version
       puts PgEasyReplicate::VERSION

data/lib/pg_easy_replicate/ddl_audit.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+
+require "pg_query"
+
+module PgEasyReplicate
+  class DDLAudit
+    extend Helper
+
+    class << self
+      def setup(group_name)
+        conn = connect_to_internal_schema
+        return if conn.table_exists?(table_name)
+
+        begin
+          conn.create_table(table_name) do
+            primary_key(:id)
+            String(:group_name, null: false)
+            String(:event_type, null: false)
+            String(:object_type)
+            String(:object_identity)
+            String(:ddl_command, text: true)
+            DateTime(:created_at, default: Sequel::CURRENT_TIMESTAMP)
+          end
+
+          create_trigger_function(conn, group_name)
+          create_event_triggers(conn, group_name)
+        rescue => e
+          abort_with("Failed to set up DDL audit: #{e.message}")
+        ensure
+          conn&.disconnect
+        end
+      end
+
+      def create(
+        group_name,
+        event_type,
+        object_type,
+        object_identity,
+        ddl_command
+      )
+        conn = connect_to_internal_schema
+        begin
+          conn[table_name].insert(
+            group_name: group_name,
+            event_type: event_type,
+            object_type: object_type,
+            object_identity: object_identity,
+            ddl_command: ddl_command,
+            created_at: Time.now.utc,
+          )
+        rescue => e
+          abort_with("Adding DDL audit entry failed: #{e.message}")
+        ensure
+          conn&.disconnect
+        end
+      end
+
+      def list_changes(group_name, limit: 100)
+        conn = connect_to_internal_schema
+        begin
+          conn[table_name]
+            .where(group_name: group_name)
+            .order(Sequel.desc(:id))
+            .limit(limit)
+            .all
+        rescue => e
+          abort_with("Listing DDL changes failed: #{e.message}")
+        ensure
+          conn&.disconnect
+        end
+      end
+
+      def apply_change(source_conn_string, target_conn_string, group_name, id)
+        ddl_queries = fetch_ddl_query(source_conn_string, group_name, id: id)
+        apply_ddl_changes(target_conn_string, ddl_queries)
+      end
+
+      def apply_all_changes(source_conn_string, target_conn_string, group_name)
+        ddl_queries = fetch_ddl_query(source_conn_string, group_name)
+        apply_ddl_changes(target_conn_string, ddl_queries)
+      end
+
+      def drop(group_name)
+        conn = connect_to_internal_schema
+        begin
+          drop_event_triggers(conn, group_name)
+          drop_trigger_function(conn, group_name)
+          conn[table_name].where(group_name: group_name).delete
+        rescue => e
+          abort_with("Dropping DDL audit failed: #{e.message}")
+        ensure
+          conn&.disconnect
+        end
+      end
+
+      private
+
+      def table_name
+        :pger_ddl_audits
+      end
+
+      def connect_to_internal_schema(conn_string = nil)
+        Query.connect(
+          connection_url: conn_string || source_db_url,
+          schema: internal_schema_name,
+        )
+      end
+
+      def create_trigger_function(conn, group_name)
+        group = PgEasyReplicate::Group.find(group_name)
+        tables = group[:table_names].split(",").map(&:strip)
+        schema_name = group[:schema_name]
+        sanitized_group_name = sanitize_identifier(group_name)
+
+        full_table_names = tables.map { |table| "#{schema_name}.#{table}" }
+        table_pattern = full_table_names.join("|")
+
+        conn.run(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}() RETURNS event_trigger AS $$
+          DECLARE
+            obj record;
+            ddl_command text;
+            affected_table text;
+          BEGIN
+            SELECT current_query() INTO ddl_command;
+
+            IF TG_EVENT = 'ddl_command_end' THEN
+              FOR obj IN SELECT * FROM pg_event_trigger_ddl_commands()
+              LOOP
+                IF obj.object_identity ~ '^(#{table_pattern})' THEN
+                  INSERT INTO #{internal_schema_name}.#{table_name} (group_name, event_type, object_type, object_identity, ddl_command)
+                  VALUES ('#{group_name}', TG_EVENT, obj.object_type, obj.object_identity, ddl_command);
+                ELSIF obj.object_type = 'index' THEN
+                  SELECT (regexp_match(ddl_command, 'ON\\s+(\\S+)'))[1] INTO affected_table;
+                  IF affected_table IN ('#{full_table_names.join("','")}') THEN
+                    INSERT INTO #{internal_schema_name}.#{table_name} (group_name, event_type, object_type, object_identity, ddl_command)
+                    VALUES ('#{group_name}', TG_EVENT, obj.object_type, obj.object_identity, ddl_command);
+                  END IF;
+                END IF;
+              END LOOP;
+            ELSIF TG_EVENT = 'sql_drop' THEN
+              FOR obj IN SELECT * FROM pg_event_trigger_dropped_objects()
+              LOOP
+                IF (obj.object_identity = ANY(ARRAY['#{full_table_names.join("','")}']) OR
+                    obj.object_identity ~ ('^' || '#{schema_name}' || '\\.(.*?)_.*$'))
+                THEN
+                  INSERT INTO #{internal_schema_name}.#{table_name} (group_name, event_type, object_type, object_identity, ddl_command)
+                  VALUES ('#{group_name}', TG_EVENT, obj.object_type, obj.object_identity, ddl_command);
+                END IF;
+              END LOOP;
+            ELSIF TG_EVENT = 'table_rewrite' THEN
+              FOR obj IN SELECT * FROM pg_event_trigger_table_rewrite_oid()
+              LOOP
+                SELECT c.relname, n.nspname INTO affected_table
+                FROM pg_class c
+                JOIN pg_namespace n ON n.oid = c.relnamespace
+                WHERE c.oid = obj.oid;
+
+                IF affected_table IN ('#{full_table_names.join("','")}') THEN
+                  INSERT INTO #{internal_schema_name}.#{table_name} (group_name, event_type, object_type, object_identity, ddl_command)
+                  VALUES ('#{group_name}', TG_EVENT, 'table', affected_table, 'table_rewrite');
+                END IF;
+              END LOOP;
+            END IF;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      rescue => e
+        abort_with("Creating DDL trigger function failed: #{e.message}")
+      end
+
+      def create_event_triggers(conn, group_name)
+        sanitized_group_name = sanitize_identifier(group_name)
+        conn.run(<<~SQL)
+          DROP EVENT TRIGGER IF EXISTS pger_ddl_trigger_#{sanitized_group_name};
+          CREATE EVENT TRIGGER pger_ddl_trigger_#{sanitized_group_name} ON ddl_command_end
+          EXECUTE FUNCTION #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}();
+
+          DROP EVENT TRIGGER IF EXISTS pger_drop_trigger_#{sanitized_group_name};
+          CREATE EVENT TRIGGER pger_drop_trigger_#{sanitized_group_name} ON sql_drop
+          EXECUTE FUNCTION #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}();
+
+          DROP EVENT TRIGGER IF EXISTS pger_table_rewrite_trigger_#{sanitized_group_name};
+          CREATE EVENT TRIGGER pger_table_rewrite_trigger_#{sanitized_group_name} ON table_rewrite
+          EXECUTE FUNCTION #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}();
+        SQL
+      rescue => e
+        abort_with("Creating event triggers failed: #{e.message}")
+      end
+
+      def drop_event_triggers(conn, group_name)
+        sanitized_group_name = sanitize_identifier(group_name)
+        conn.run(<<~SQL)
+          DROP EVENT TRIGGER IF EXISTS pger_ddl_trigger_#{sanitized_group_name};
+          DROP EVENT TRIGGER IF EXISTS pger_drop_trigger_#{sanitized_group_name};
+          DROP EVENT TRIGGER IF EXISTS pger_table_rewrite_trigger_#{sanitized_group_name};
+        SQL
+      rescue => e
+        abort_with("Dropping event triggers failed: #{e.message}")
+      end
+
+      def drop_trigger_function(conn, group_name)
+        sanitized_group_name = sanitize_identifier(group_name)
+        conn.run(
+          "DROP FUNCTION IF EXISTS #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}();",
+        )
+      rescue => e
+        abort_with("Dropping trigger function failed: #{e.message}")
+      end
+
+      def self.extract_table_info(sql)
+        parsed = PgQuery.parse(sql)
+        stmt = parsed.tree.stmts.first.stmt
+
+        case stmt
+        when PgQuery::CreateStmt, PgQuery::IndexStmt, PgQuery::AlterTableStmt
+          schema_name = stmt.relation.schemaname || "public"
+          table_name = stmt.relation.relname
+          "#{schema_name}.#{table_name}"
+        end
+      rescue PgQuery::ParseError
+        nil
+      end
+
+      def sanitize_identifier(identifier)
+        identifier.gsub(/[^a-zA-Z0-9_]/, "_")
+      end
+
+      def fetch_ddl_query(source_conn_string, group_name, id: nil)
+        source_conn = connect_to_internal_schema(source_conn_string)
+        begin
+          query = source_conn[table_name].where(group_name: group_name)
+          query = query.where(id: id) if id
+          result = query.order(:id).select_map(:ddl_command)
+          result.uniq
+        rescue => e
+          abort_with("Fetching DDL queries failed: #{e.message}")
+        ensure
+          source_conn&.disconnect
+        end
+      end
+
+      def apply_ddl_changes(target_conn_string, ddl_queries)
+        target_conn = Query.connect(connection_url: target_conn_string)
+        begin
+          ddl_queries.each do |query|
+            target_conn.run(query)
+          rescue => e
+            abort_with(
+              "Error executing DDL command: #{query}. Error: #{e.message}",
+            )
+          end
+        rescue => e
+          abort_with("Applying DDL changes failed: #{e.message}")
+        ensure
+          target_conn&.disconnect
+        end
+      end
+    end
+  end
+end

data/lib/pg_easy_replicate/ddl_manager.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module PgEasyReplicate
+  module DDLManager
+    extend Helper
+
+    class << self
+      def setup_ddl_tracking(
+        group_name:, conn_string: source_db_url,
+        schema: "public"
+      )
+        DDLAudit.setup(group_name)
+      end
+
+      def cleanup_ddl_tracking(
+        group_name:, conn_string: source_db_url,
+        schema: "public"
+      )
+        DDLAudit.drop(group_name)
+      end
+
+      def list_ddl_changes(
+        group_name:, conn_string: source_db_url,
+        schema: "public",
+        limit: 100
+      )
+        DDLAudit.list_changes(group_name, limit: limit)
+      end
+
+      def apply_ddl_change(
+        group_name:, id:, source_conn_string: source_db_url,
+        target_conn_string: target_db_url,
+        schema: "public"
+      )
+        DDLAudit.apply_change(
+          source_conn_string,
+          target_conn_string,
+          group_name,
+          id,
+        )
+      end
+
+      def apply_all_ddl_changes(
+        group_name:, source_conn_string: source_db_url,
+        target_conn_string: target_db_url,
+        schema: "public"
+      )
+        DDLAudit.apply_all_changes(
+          source_conn_string,
+          target_conn_string,
+          group_name,
+        )
+      end
+    end
+  end
+end

data/lib/pg_easy_replicate/orchestrate.rb

@@ -15,7 +15,7 @@ module PgEasyReplicate
             schema: schema_name,
             conn_string: source_db_url,
             list: options[:tables],
-            exclude_list: options[:exclude_tables]
+            exclude_list: options[:exclude_tables],
           )
 
         if options[:recreate_indices_post_copy]
@@ -52,25 +52,19 @@ module PgEasyReplicate
           started_at: Time.now.utc,
           recreate_indices_post_copy: options[:recreate_indices_post_copy],
         )
-      rescue => e
-        stop_sync(
-          group_name: options[:group_name],
-          source_conn_string: source_db_url,
-          target_conn_string: target_db_url,
-        )
 
-        if Group.find(options[:group_name])
-          Group.update(group_name: options[:group_name], failed_at: Time.now)
-        else
-          Group.create(
-            name: options[:group_name],
-            table_names: tables.join(","),
-            schema_name: schema_name,
-            started_at: Time.now.utc,
-            failed_at: Time.now.utc,
+        if options[:track_ddl]
+          DDLManager.setup_ddl_tracking(
+            conn_string: source_db_url,
+            group_name: options[:group_name],
+            schema: schema_name,
           )
         end
-
+      rescue => e
+        stop_sync(group_name: options[:group_name])
+        if Group.find(options[:group_name])
+          Group.update(name: options[:group_name], failed_at: Time.now.utc)
+        end
         abort_with("Starting sync failed: #{e.message}")
       end
 
@@ -180,8 +174,8 @@ module PgEasyReplicate
 
       def stop_sync(
         group_name:,
-        source_conn_string: source_db_url,
-        target_conn_string: target_db_url
+        source_conn_string: nil,
+        target_conn_string: nil
       )
         logger.info(
           "Stopping sync",
@@ -192,29 +186,34 @@ module PgEasyReplicate
         )
         drop_publication(
           group_name: group_name,
-          conn_string: source_conn_string,
+          conn_string: source_conn_string || source_db_url,
         )
         drop_subscription(
           group_name: group_name,
-          target_conn_string: target_conn_string,
+          target_conn_string: target_conn_string || target_db_url,
         )
       rescue => e
-        raise "Unable to stop sync user: #{e.message}"
+        abort_with("Unable to stop sync: #{e.message}")
       end
 
       def switchover(
         group_name:,
-        source_conn_string: source_db_url,
-        target_conn_string: target_db_url,
         lag_delta_size: nil,
-        skip_vacuum_analyze: false
+        skip_vacuum_analyze: false,
+        source_conn_string: nil,
+        target_conn_string: nil
       )
         group = Group.find(group_name)
+        abort_with("Group not found: #{group_name}") unless group
+
         tables_list = group[:table_names].split(",")
 
+        source_conn = source_conn_string || source_db_url
+        target_conn = target_conn_string || target_db_url
+
         unless skip_vacuum_analyze
           run_vacuum_analyze(
-            conn_string: target_conn_string,
+            conn_string: target_conn,
             tables: tables_list,
             schema: group[:schema_name],
           )
@@ -225,39 +224,35 @@ module PgEasyReplicate
         if group[:recreate_indices_post_copy]
           IndexManager.wait_for_replication_completion(group_name: group_name)
           IndexManager.recreate_indices(
-            source_conn_string: source_db_url,
-            target_conn_string: target_db_url,
+            source_conn_string: source_conn,
+            target_conn_string: target_conn,
             tables: tables_list,
             schema: group[:schema_name],
           )
         end
 
-        # Watch for lag again, because it could've grown during index recreation
         watch_lag(group_name: group_name, lag: lag_delta_size || DEFAULT_LAG)
 
         revoke_connections_on_source_db(group_name)
         wait_for_remaining_catchup(group_name)
-        refresh_sequences(
-          conn_string: target_conn_string,
-          schema: group[:schema_name],
-        )
+        refresh_sequences(conn_string: target_conn, schema: group[:schema_name])
         mark_switchover_complete(group_name)
-        # Run vacuum analyze to refresh the planner post switchover
+
         unless skip_vacuum_analyze
           run_vacuum_analyze(
-            conn_string: target_conn_string,
+            conn_string: target_conn,
             tables: tables_list,
             schema: group[:schema_name],
           )
         end
+
         drop_subscription(
           group_name: group_name,
-          target_conn_string: target_conn_string,
+          target_conn_string: target_conn,
         )
       rescue => e
         restore_connections_on_source_db(group_name)
-
-        abort_with("Switchover sync failed: #{e.message}")
+        abort_with("Switchover failed: #{e.message}")
       end
 
       def watch_lag(group_name:, wait_time: DEFAULT_WAIT, lag: DEFAULT_LAG)

data/lib/pg_easy_replicate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEasyReplicate
-  VERSION = "0.2.7"
+  VERSION = "0.3.1"
 end

data/lib/pg_easy_replicate.rb

@@ -6,6 +6,7 @@ require "pg"
 require "sequel"
 require "open3"
 require "English"
+require "pg_query"
 
 require "pg_easy_replicate/helper"
 require "pg_easy_replicate/version"
@@ -15,6 +16,8 @@ require "pg_easy_replicate/orchestrate"
 require "pg_easy_replicate/stats"
 require "pg_easy_replicate/group"
 require "pg_easy_replicate/cli"
+require "pg_easy_replicate/ddl_audit"
+require "pg_easy_replicate/ddl_manager"
 
 Sequel.default_timezone = :utc
 module PgEasyReplicate
@@ -200,6 +203,14 @@ module PgEasyReplicate
             logger.info("Dropping replication user on target database")
             drop_user(conn_string: target_db_url)
           end
+          -> do
+            if options[:everything]
+              PgEasyReplicate::DDLManager.cleanup_ddl_tracking(
+                conn_string: source_db_url,
+                group_name: options[:group_name],
+              )
+            end
+          end
         end,
       ]
 

data/scripts/e2e-start.sh

@@ -10,9 +10,17 @@ export SOURCE_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhos
 export TARGET_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhost:5433/postgres-db"
 export PGPASSWORD='james-bond123@7!'"'"''"'"'3aaR'
 
-# Config check, Bootstrap and cleanup
 echo "===== Performing Bootstrap and cleanup"
 bundle exec bin/pg_easy_replicate bootstrap -g cluster-1 --copy-schema
-bundle exec bin/pg_easy_replicate start_sync -g cluster-1 -s public --recreate-indices-post-copy
+bundle exec bin/pg_easy_replicate start_sync -g cluster-1 -s public --recreate-indices-post-copy --track-ddl
 bundle exec bin/pg_easy_replicate stats -g cluster-1
+
+echo "===== Applying DDL change"
+psql $SOURCE_DB_URL -c "ALTER TABLE public.pgbench_accounts ADD COLUMN test_column VARCHAR(255)"
+
+echo "===== Applying DDL changes"
+echo "Y" | bundle exec bin/pg_easy_replicate apply_ddl_change -g cluster-1
+
+# Switchover
+echo "===== Performing switchover"
 bundle exec bin/pg_easy_replicate switchover -g cluster-1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_easy_replicate
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.3.1
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-09 00:00:00.000000000 Z
+date: 2024-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.5.3
+- !ruby/object:Gem::Dependency
+  name: pg_query
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.0
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -47,7 +61,7 @@ dependencies:
         version: '5.69'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.83'
+        version: '5.84'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -57,7 +71,7 @@ dependencies:
         version: '5.69'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.83'
+        version: '5.84'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -274,6 +288,8 @@ files:
 - docker-compose.yml
 - lib/pg_easy_replicate.rb
 - lib/pg_easy_replicate/cli.rb
+- lib/pg_easy_replicate/ddl_audit.rb
+- lib/pg_easy_replicate/ddl_manager.rb
 - lib/pg_easy_replicate/group.rb
 - lib/pg_easy_replicate/helper.rb
 - lib/pg_easy_replicate/index_manager.rb