pg_easy_replicate 0.2.6 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 81be66e36a67bb505548d91dabf061ad83724c6d849bb611701769ffe3e7abc8
-  data.tar.gz: 2223eb05e483c2481a68b157e75f9aca9fd62f5205de8fe5664ce91364bbaf73
+  metadata.gz: 8b12c7664573414ae5b4421e16c2a4d603a1550abd52cd677fa97c0a3f4187d3
+  data.tar.gz: 03baf5fa2d60d841475024d8e6a4e822546a5f5180ddcdead5ee33ed89bc472b
 SHA512:
-  metadata.gz: e3a93263182fc9c68c5b417d5512b021753ef356efb2e6c11ce2098e4c49b2ea41f21533f9aa590032a8eee0663a8ded1633f2d87399a11bd4c150da56c8f19e
-  data.tar.gz: a24dc626daa748db907b07f8542008f2d5b6de558f6b80aa6f5879435a3eac92a6282e3a6bebc6d637502967adf3b77e9c57d8e67bd3a58136ca2c1f871b85be
+  metadata.gz: c18024373cdf4313723599eaee6f57d84b99ff52afffbbab3a393bae9e453ba43d300f8ec3424396af12fcbf40de46c92846110777a7a6d4ebb6fe9adae47f5f
+  data.tar.gz: 8f4e8dc80d25b072b6e44a4e17fc6d367fe27368a9643e4e0cc4a39dd846cd1abf247aedbc6e887b1285293939f967131e0d5254ae53751f584af2e13a72c783

data/CHANGELOG.md

@@ -1,4 +1,10 @@
-## [0.2.4] - 2024-04-14
+## [0.2.6] - 2024-06-04
+
+- Quote table name in the VACUUM SQL - #118
+- Exclude tables created by extensions - #120
+- Use db user when adding tables to replication - #130
+
+## [0.2.5] - 2024-04-14
 
 - List only permanent tables - #113
 

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    pg_easy_replicate (0.2.6)
+    pg_easy_replicate (0.3.0)
       ougai (~> 2.0.0)
       pg (~> 1.5.3)
-      sequel (>= 5.69, < 5.82)
+      pg_query (~> 5.1.0)
+      sequel (>= 5.69, < 5.83)
       thor (>= 1.2.2, < 1.4.0)
 
 GEM
@@ -14,6 +15,12 @@ GEM
     bigdecimal (3.1.8)
     coderay (1.1.3)
     diff-lcs (1.5.1)
+    google-protobuf (4.28.0-arm64-darwin)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.28.0-x86_64-linux)
+      bigdecimal
+      rake (>= 13)
     haml (6.1.1)
       temple (>= 0.8.2)
       thor
@@ -24,11 +31,13 @@ GEM
     oj (3.14.3)
     ougai (2.0.0)
       oj (~> 3.10)
-    parallel (1.24.0)
-    parser (3.3.2.0)
+    parallel (1.25.1)
+    parser (3.3.3.0)
       ast (~> 2.4.1)
       racc
-    pg (1.5.6)
+    pg (1.5.7)
+    pg_query (5.1.0)
+      google-protobuf (>= 3.22.3)
     prettier_print (1.2.1)
     pry (0.14.2)
       coderay (~> 1.1)
@@ -38,8 +47,8 @@ GEM
     rake (13.2.1)
     rbs (3.1.0)
     regexp_parser (2.9.2)
-    rexml (3.2.8)
-      strscan (>= 3.0.9)
+    rexml (3.3.6)
+      strscan
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
@@ -72,7 +81,7 @@ GEM
       rubocop (~> 1.41)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-    rubocop-performance (1.21.0)
+    rubocop-performance (1.21.1)
       rubocop (>= 1.48.1, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-rake (0.6.0)
@@ -85,7 +94,7 @@ GEM
     rubocop-rspec_rails (2.28.2)
       rubocop (~> 1.40)
     ruby-progressbar (1.13.0)
-    sequel (5.81.0)
+    sequel (5.82.0)
       bigdecimal
     strscan (3.1.0)
     syntax_tree (6.2.0)

data/README.md

@@ -69,7 +69,12 @@ All [Logical Replication Restrictions](https://www.postgresql.org/docs/current/l
 
 ## Usage
 
-Ensure `SOURCE_DB_URL` and `TARGET_DB_URL` are present as environment variables in the runtime environment. The URL are of the postgres connection string format. Example:
+Ensure `SOURCE_DB_URL` and `TARGET_DB_URL` are present as environment variables in the runtime environment.
+
+- `SOURCE_DB_URL` = The database that you want to replicate FROM.
+- `TARGET_DB_URL` = The database that you want to replicate TO.
+
+The URL should be in postgres connection string format. Example:
 
 ```bash
 $ export SOURCE_DB_URL="postgres://USERNAME:PASSWORD@localhost:5432/DATABASE_NAME"
@@ -84,6 +89,12 @@ You can extend the default timeout by setting the following environment variable
 $ export PG_EASY_REPLICATE_STATEMENT_TIMEOUT="10s" # default 5s
 ```
 
+You can get additional debug logging by adding the following environment variable
+
+```bash
+$ export DEBUG="true"
+```
+
 Any `pg_easy_replicate` command can be run the same way with the docker image as well. As long the container is running in an environment where it has access to both the databases. Example
 
 ```bash
@@ -165,12 +176,62 @@ Once the bootstrap is complete, you can start the sync. Starting the sync sets u
 **NOTE**: Start sync by default will drop all indices in the target database for performance reasons. And will automatically re-add the indices during `switchover`. It is turned on by default and you can opt out of this with `--no-recreate-indices-post-copy`
 
 ```bash
-$ pg_easy_replicate start_sync --group-name database-cluster-1
+$ pg_easy_replicate start_sync --group-name database-cluster-1 [-d <track-ddl>]
 
 {"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":22113,"level":30,"time":"2023-06-19T15:54:54.874-04:00","v":0,"msg":"Setting up publication","publication_name":"pger_publication_database_cluster_1","version":"0.1.0"}
 ...
 ```
 
+### DDL Changes Management
+
+`pg_easy_replicate` now supports tracking and applying DDL (Data Definition Language) changes between the source and target databases. To track DDLs you can pass `-track-ddl` to `start_sync`.
+
+This feature ensures that most schema changes made to the source database tables that are being replicated during the replication process are tracked, so that you can apply them at your will before or after switchover.
+
+#### Listing DDL Changes
+
+To view the DDL changes that have been tracked:
+
+```bash
+$ pg_easy_replicate list_ddl_changes -g <group-name> [-l <limit>]
+```
+
+This command will display a list of DDL changes in JSON format;
+
+```
+[
+  {
+    "id": 1,
+    "group_name": "cluster-1",
+    "event_type": "ddl_command_end",
+    "object_type": "table",
+    "object_identity": "public.pgbench_accounts",
+    "ddl_command": "ALTER TABLE public.pgbench_accounts ADD COLUMN test_column VARCHAR(255)",
+    "created_at": "2024-08-31 15:42:33 UTC"
+  }
+]
+```
+
+#### Applying DDL Changes
+
+`pg_easy_replicate` won't automatically apply the changes for you. To apply the tracked DDL changes to the target database:
+
+```bash
+$ pg_easy_replicate apply_ddl_change -g <group-name> [-i <change-id>]
+```
+
+If you specify a change ID with the `-i` option, only that specific change will be applied. If you don't specify an ID, you'll be prompted to apply all pending changes.
+
+```bash
+$ pg_easy_replicate apply_ddl_change -g cluster-1
+The following DDL changes will be applied:
+ID: 1, Type: table, Command: ALTER TABLE public.pgbench_accounts ADD COLUMN test_column VARCHAR(255)...
+
+Do you want to apply all these changes? (y/n): y
+...
+All pending DDL changes applied successfully.
+```
+
 ### Stats
 
 You can inspect or watch stats any time during the sync process. The stats give you an idea of when the sync started, current flush/write lag, how many tables are in `replicating`, `copying` or other stages, and more.
@@ -247,6 +308,30 @@ $ pg_easy_replicate switchover  --group-name database-cluster-2
 ...
 ```
 
+## Exclude tables from replication
+
+By default all tables are added for replication but you can exclude tables if necessary. Example
+
+```bash
+...
+$ pg_easy_replicate bootstrap --group-name database-cluster-1 --copy-schema
+$ pg_easy_replicate start_sync --group-name database-cluster-1 --schema-name public --exclude_tables "events"
+...
+```
+
+### Cleanup
+
+Use `cleanup` if you want to remove all bootstrapped data for the specified group. Additionally you can pass `-e` or `--everything` in order to clean up all schema changes for bootstrapped tables, users and any publication/subscription data.
+
+```bash
+$ pg_easy_replicate cleanup  --group-name database-cluster-1 --everything
+
+{"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":24192,"level":30,"time":"2023-06-19T16:05:23.033-04:00","v":0,"msg":"Dropping groups table","version":"0.1.0"}
+
+{"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":24192,"level":30,"time":"2023-06-19T16:05:23.033-04:00","v":0,"msg":"Dropping schema","version":"0.1.0"}
+...
+```
+
 ## Switchover strategies with minimal downtime
 
 For minimal downtime, it'd be best to watch/tail the stats and wait until `switchover_completed_at` is updated with a timestamp. Once that happens you can perform any of the following strategies. Note: These are just suggestions and `pg_easy_replicate` doesn't provide any functionalities for this.

data/docker-compose.yml

@@ -9,6 +9,7 @@ services:
       POSTGRES_PASSWORD: james-bond123@7!'3aaR
       POSTGRES_DB: postgres-db
     command: >
+      -c max_connections=200
       -c wal_level=logical
       -c ssl=on
       -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
@@ -25,6 +26,7 @@ services:
       POSTGRES_PASSWORD: james-bond123@7!'3aaR
       POSTGRES_DB: postgres-db
     command: >
+      -c max_connections=200
       -c wal_level=logical
       -c ssl=on
       -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem

data/lib/pg_easy_replicate/cli.rb

@@ -21,6 +21,11 @@ module PgEasyReplicate
                   default: "",
                   desc:
                     "Comma separated list of table names. Default: All tables"
+    method_option :exclude_tables,
+                  aliases: "-e",
+                  default: "",
+                  desc:
+                    "Comma separated list of table names to exclude. Default: None"
     method_option :schema_name,
                   aliases: "-s",
                   desc:
@@ -30,6 +35,7 @@ module PgEasyReplicate
         special_user_role: options[:special_user_role],
         copy_schema: options[:copy_schema],
         tables: options[:tables],
+        exclude_tables: options[:exclude_tables],
         schema_name: options[:schema_name],
       )
 
@@ -48,6 +54,11 @@ module PgEasyReplicate
                   aliases: "-c",
                   boolean: true,
                   desc: "Copy schema to the new database"
+    method_option :track_ddl,
+                  aliases: "-d",
+                  type: :boolean,
+                  default: false,
+                  desc: "Enable DDL tracking for the group"
     desc "bootstrap",
          "Sets up temporary tables for information required during runtime"
     def bootstrap
@@ -77,11 +88,6 @@ module PgEasyReplicate
                   aliases: "-g",
                   required: true,
                   desc: "Name of the group to provision"
-    method_option :group_name,
-                  aliases: "-g",
-                  required: true,
-                  desc:
-                    "Name of the grouping for this collection of source and target DB"
     method_option :schema_name,
                   aliases: "-s",
                   desc:
@@ -91,12 +97,21 @@ module PgEasyReplicate
                   default: "",
                   desc:
                     "Comma separated list of table names. Default: All tables"
+    method_option :exclude_tables,
+                  aliases: "-e",
+                  default: "",
+                  desc:
+                    "Comma separated list of table names to exclude. Default: None"
     method_option :recreate_indices_post_copy,
                   type: :boolean,
-                  default: true,
+                  default: false,
                   aliases: "-r",
                   desc:
                     "Drop all non-primary indices before copy and recreate them post-copy"
+    method_option :track_ddl,
+                  type: :boolean,
+                  default: false,
+                  desc: "Enable DDL tracking for the group"
     def start_sync
       PgEasyReplicate::Orchestrate.start_sync(options)
     end
@@ -111,7 +126,7 @@ module PgEasyReplicate
       PgEasyReplicate::Orchestrate.stop_sync(group_name: options[:group_name])
     end
 
-    desc "switchover ",
+    desc "switchover",
          "Puts the source database in read only mode after all the data is flushed and written"
     method_option :group_name,
                   aliases: "-g",
@@ -119,21 +134,18 @@ module PgEasyReplicate
                   desc: "Name of the group previously provisioned"
     method_option :lag_delta_size,
                   aliases: "-l",
-                  desc: "The size of the lag to watch for before switchover. Default 200KB."
+                  desc:
+                    "The size of the lag to watch for before switchover. Default 200KB."
     method_option :skip_vacuum_analyze,
                   type: :boolean,
                   default: false,
                   aliases: "-s",
                   desc: "Skip vacuum analyzing tables before switchover."
-    # method_option :bi_directional,
-    #               aliases: "-b",
-    #               desc:
-    #                 "Setup replication from target database to source database"
     def switchover
       PgEasyReplicate::Orchestrate.switchover(
         group_name: options[:group_name],
         lag_delta_size: options[:lag_delta_size],
-        skip_vacuum_analyze: options[:skip_vacuum_analyze]
+        skip_vacuum_analyze: options[:skip_vacuum_analyze],
       )
     end
 
@@ -151,6 +163,71 @@ module PgEasyReplicate
       end
     end
 
+    desc "list_ddl_changes", "Lists recent DDL changes in the source database"
+    method_option :group_name,
+                  aliases: "-g",
+                  required: true,
+                  desc: "Name of the group"
+    method_option :limit,
+                  aliases: "-l",
+                  type: :numeric,
+                  default: 100,
+                  desc: "Limit the number of DDL changes to display"
+    def list_ddl_changes
+      changes =
+        PgEasyReplicate::DDLManager.list_ddl_changes(
+          group_name: options[:group_name],
+          limit: options[:limit],
+        )
+      puts JSON.pretty_generate(changes)
+    end
+
+    desc "apply_ddl_change", "Applies DDL changes to the target database"
+    method_option :group_name,
+                  aliases: "-g",
+                  required: true,
+                  desc: "Name of the group"
+    method_option :id,
+                  aliases: "-i",
+                  type: :numeric,
+                  desc:
+                    "ID of the specific DDL change to apply. If not provided, all changes will be applied."
+    def apply_ddl_change
+      if options[:id]
+        PgEasyReplicate::DDLManager.apply_ddl_change(
+          group_name: options[:group_name],
+          id: options[:id],
+        )
+        puts "DDL change with ID #{options[:id]} applied successfully."
+      else
+        changes =
+          PgEasyReplicate::DDLManager.list_ddl_changes(
+            group_name: options[:group_name],
+          )
+        if changes.empty?
+          puts "No pending DDL changes to apply."
+          return
+        end
+
+        puts "The following DDL changes will be applied:"
+        changes.each do |change|
+          puts "ID: #{change[:id]}, Type: #{change[:object_type]}, Command: #{change[:ddl_command]}"
+        end
+        puts ""
+        print("Do you want to apply all these changes? (y/n): ")
+        confirmation = $stdin.gets.chomp.downcase
+
+        if confirmation == "y"
+          PgEasyReplicate::DDLManager.apply_all_ddl_changes(
+            group_name: options[:group_name],
+          )
+          puts "All pending DDL changes applied successfully."
+        else
+          puts "Operation cancelled."
+        end
+      end
+    end
+
     desc "version", "Prints the version"
     def version
       puts PgEasyReplicate::VERSION

data/lib/pg_easy_replicate/ddl_audit.rb

@@ -0,0 +1,256 @@
+# frozen_string_literal: true
+
+require "pg_query"
+
+module PgEasyReplicate
+  class DDLAudit
+    extend Helper
+
+    class << self
+      def setup(group_name)
+        conn = connect_to_internal_schema
+        return if conn.table_exists?(table_name)
+
+        begin
+          conn.create_table(table_name) do
+            primary_key(:id)
+            String(:group_name, null: false)
+            String(:event_type, null: false)
+            String(:object_type)
+            String(:object_identity)
+            String(:ddl_command, text: true)
+            DateTime(:created_at, default: Sequel::CURRENT_TIMESTAMP)
+          end
+
+          create_trigger_function(conn, group_name)
+          create_event_triggers(conn, group_name)
+        rescue => e
+          abort_with("Failed to set up DDL audit: #{e.message}")
+        ensure
+          conn&.disconnect
+        end
+      end
+
+      def create(
+        group_name,
+        event_type,
+        object_type,
+        object_identity,
+        ddl_command
+      )
+        conn = connect_to_internal_schema
+        begin
+          conn[table_name].insert(
+            group_name: group_name,
+            event_type: event_type,
+            object_type: object_type,
+            object_identity: object_identity,
+            ddl_command: ddl_command,
+            created_at: Time.now.utc,
+          )
+        rescue => e
+          abort_with("Adding DDL audit entry failed: #{e.message}")
+        ensure
+          conn&.disconnect
+        end
+      end
+
+      def list_changes(group_name, limit: 100)
+        conn = connect_to_internal_schema
+        begin
+          conn[table_name]
+            .where(group_name: group_name)
+            .order(Sequel.desc(:id))
+            .limit(limit)
+            .all
+        rescue => e
+          abort_with("Listing DDL changes failed: #{e.message}")
+        ensure
+          conn&.disconnect
+        end
+      end
+
+      def apply_change(source_conn_string, target_conn_string, group_name, id)
+        ddl_queries = fetch_ddl_query(source_conn_string, group_name, id: id)
+        apply_ddl_changes(target_conn_string, ddl_queries)
+      end
+
+      def apply_all_changes(source_conn_string, target_conn_string, group_name)
+        ddl_queries = fetch_ddl_query(source_conn_string, group_name)
+        apply_ddl_changes(target_conn_string, ddl_queries)
+      end
+
+      def drop(group_name)
+        conn = connect_to_internal_schema
+        begin
+          drop_event_triggers(conn, group_name)
+          drop_trigger_function(conn, group_name)
+          conn[table_name].where(group_name: group_name).delete
+        rescue => e
+          abort_with("Dropping DDL audit failed: #{e.message}")
+        ensure
+          conn&.disconnect
+        end
+      end
+
+      private
+
+      def table_name
+        :pger_ddl_audits
+      end
+
+      def connect_to_internal_schema(conn_string = nil)
+        Query.connect(
+          connection_url: conn_string || source_db_url,
+          schema: internal_schema_name,
+        )
+      end
+
+      def create_trigger_function(conn, group_name)
+        group = PgEasyReplicate::Group.find(group_name)
+        tables = group[:table_names].split(",").map(&:strip)
+        schema_name = group[:schema_name]
+        sanitized_group_name = sanitize_identifier(group_name)
+
+        full_table_names = tables.map { |table| "#{schema_name}.#{table}" }
+        puts "full_table_names: #{full_table_names}"
+        conn.run(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}() RETURNS event_trigger AS $$
+          DECLARE
+            obj record;
+            ddl_command text;
+            affected_table text;
+          BEGIN
+            SELECT current_query() INTO ddl_command;
+
+            IF TG_EVENT = 'ddl_command_end' THEN
+              FOR obj IN SELECT * FROM pg_event_trigger_ddl_commands()
+              LOOP
+                IF obj.object_type = 'table' AND obj.object_identity = ANY(ARRAY['#{full_table_names.join("','")}']) THEN
+                  INSERT INTO #{internal_schema_name}.#{table_name} (group_name, event_type, object_type, object_identity, ddl_command)
+                  VALUES ('#{group_name}', TG_EVENT, obj.object_type, obj.object_identity, ddl_command);
+                ELSIF obj.object_type = 'index' THEN
+                  SELECT (regexp_match(ddl_command, 'ON\\s+(\\S+)'))[1] INTO affected_table;
+                  IF affected_table = ANY(ARRAY['#{full_table_names.join("','")}']) THEN
+                    INSERT INTO #{internal_schema_name}.#{table_name} (group_name, event_type, object_type, object_identity, ddl_command)
+                    VALUES ('#{group_name}', TG_EVENT, obj.object_type, obj.object_identity, ddl_command);
+                  END IF;
+                END IF;
+              END LOOP;
+            ELSIF TG_EVENT = 'sql_drop' THEN
+              FOR obj IN SELECT * FROM pg_event_trigger_dropped_objects()
+              LOOP
+                IF obj.object_type IN ('table', 'index') AND
+                   (obj.object_identity = ANY(ARRAY['#{full_table_names.join("','")}']) OR
+                    obj.object_identity ~ ('^' || '#{schema_name}' || '\\.(.*?)_.*$'))
+                THEN
+                  INSERT INTO #{internal_schema_name}.#{table_name} (group_name, event_type, object_type, object_identity, ddl_command)
+                  VALUES ('#{group_name}', TG_EVENT, obj.object_type, obj.object_identity, ddl_command);
+                END IF;
+              END LOOP;
+            ELSIF TG_EVENT = 'table_rewrite' THEN
+              FOR obj IN SELECT * FROM pg_event_trigger_table_rewrite_oid()
+              LOOP
+                IF obj.oid::regclass::text = ANY(ARRAY['#{full_table_names.join("','")}']) THEN
+                  INSERT INTO #{internal_schema_name}.#{table_name} (group_name, event_type, object_type, object_identity, ddl_command)
+                  VALUES ('#{group_name}', TG_EVENT, 'table', obj.oid::regclass::text, ddl_command);
+                END IF;
+              END LOOP;
+            END IF;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      rescue => e
+        abort_with("Creating DDL trigger function failed: #{e.message}")
+      end
+
+      def create_event_triggers(conn, group_name)
+        sanitized_group_name = sanitize_identifier(group_name)
+        conn.run(<<~SQL)
+          DROP EVENT TRIGGER IF EXISTS pger_ddl_trigger_#{sanitized_group_name};
+          CREATE EVENT TRIGGER pger_ddl_trigger_#{sanitized_group_name} ON ddl_command_end
+          EXECUTE FUNCTION #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}();
+
+          DROP EVENT TRIGGER IF EXISTS pger_drop_trigger_#{sanitized_group_name};
+          CREATE EVENT TRIGGER pger_drop_trigger_#{sanitized_group_name} ON sql_drop
+          EXECUTE FUNCTION #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}();
+
+          DROP EVENT TRIGGER IF EXISTS pger_table_rewrite_trigger_#{sanitized_group_name};
+          CREATE EVENT TRIGGER pger_table_rewrite_trigger_#{sanitized_group_name} ON table_rewrite
+          EXECUTE FUNCTION #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}();
+        SQL
+      rescue => e
+        abort_with("Creating event triggers failed: #{e.message}")
+      end
+
+      def drop_event_triggers(conn, group_name)
+        sanitized_group_name = sanitize_identifier(group_name)
+        conn.run(<<~SQL)
+          DROP EVENT TRIGGER IF EXISTS pger_ddl_trigger_#{sanitized_group_name};
+          DROP EVENT TRIGGER IF EXISTS pger_drop_trigger_#{sanitized_group_name};
+          DROP EVENT TRIGGER IF EXISTS pger_table_rewrite_trigger_#{sanitized_group_name};
+        SQL
+      rescue => e
+        abort_with("Dropping event triggers failed: #{e.message}")
+      end
+
+      def drop_trigger_function(conn, group_name)
+        sanitized_group_name = sanitize_identifier(group_name)
+        conn.run(
+          "DROP FUNCTION IF EXISTS #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}();",
+        )
+      rescue => e
+        abort_with("Dropping trigger function failed: #{e.message}")
+      end
+
+      def self.extract_table_info(sql)
+        parsed = PgQuery.parse(sql)
+        stmt = parsed.tree.stmts.first.stmt
+
+        case stmt
+        when PgQuery::CreateStmt, PgQuery::IndexStmt, PgQuery::AlterTableStmt
+          schema_name = stmt.relation.schemaname || "public"
+          table_name = stmt.relation.relname
+          "#{schema_name}.#{table_name}"
+        end
+      rescue PgQuery::ParseError
+        nil
+      end
+
+      def sanitize_identifier(identifier)
+        identifier.gsub(/[^a-zA-Z0-9_]/, "_")
+      end
+
+      def fetch_ddl_query(source_conn_string, group_name, id: nil)
+        source_conn = connect_to_internal_schema(source_conn_string)
+        begin
+          query = source_conn[table_name].where(group_name: group_name)
+          query = query.where(id: id) if id
+          result = query.order(:id).select_map(:ddl_command)
+          result.uniq
+        rescue => e
+          abort_with("Fetching DDL queries failed: #{e.message}")
+        ensure
+          source_conn&.disconnect
+        end
+      end
+
+      def apply_ddl_changes(target_conn_string, ddl_queries)
+        target_conn = Query.connect(connection_url: target_conn_string)
+        begin
+          ddl_queries.each do |query|
+            target_conn.run(query)
+          rescue => e
+            abort_with(
+              "Error executing DDL command: #{query}. Error: #{e.message}",
+            )
+          end
+        rescue => e
+          abort_with("Applying DDL changes failed: #{e.message}")
+        ensure
+          target_conn&.disconnect
+        end
+      end
+    end
+  end
+end

data/lib/pg_easy_replicate/ddl_manager.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module PgEasyReplicate
+  module DDLManager
+    extend Helper
+
+    class << self
+      def setup_ddl_tracking(
+        group_name:, conn_string: source_db_url,
+        schema: "public"
+      )
+        DDLAudit.setup(group_name)
+      end
+
+      def cleanup_ddl_tracking(
+        group_name:, conn_string: source_db_url,
+        schema: "public"
+      )
+        DDLAudit.drop(group_name)
+      end
+
+      def list_ddl_changes(
+        group_name:, conn_string: source_db_url,
+        schema: "public",
+        limit: 100
+      )
+        DDLAudit.list_changes(group_name, limit: limit)
+      end
+
+      def apply_ddl_change(
+        group_name:, id:, source_conn_string: source_db_url,
+        target_conn_string: target_db_url,
+        schema: "public"
+      )
+        DDLAudit.apply_change(
+          source_conn_string,
+          target_conn_string,
+          group_name,
+          id,
+        )
+      end
+
+      def apply_all_ddl_changes(
+        group_name:, source_conn_string: source_db_url,
+        target_conn_string: target_db_url,
+        schema: "public"
+      )
+        DDLAudit.apply_all_changes(
+          source_conn_string,
+          target_conn_string,
+          group_name,
+        )
+      end
+    end
+  end
+end

data/lib/pg_easy_replicate/helper.rb

@@ -73,14 +73,18 @@ module PgEasyReplicate
       abort(msg)
     end
 
-    def determine_tables(conn_string:, list: "", schema: nil)
+    def determine_tables(conn_string:, list: "", exclude_list: "", schema: nil)
       schema ||= "public"
-
-      tables = list&.split(",") || []
-      if tables.size > 0
-        tables
+      
+      tables = convert_to_array(list)
+      exclude_tables = convert_to_array(exclude_list)
+      validate_table_lists(tables, exclude_tables, schema)
+
+      if tables.empty?
+        all_tables = list_all_tables(schema: schema, conn_string: conn_string)
+        all_tables - (exclude_tables + %w[spatial_ref_sys])
       else
-        list_all_tables(schema: schema, conn_string: conn_string) - %w[ spatial_ref_sys ]
+        tables
       end
     end
 
@@ -101,5 +105,24 @@ module PgEasyReplicate
         .map(&:values)
         .flatten
     end
+
+    def convert_to_array(input)
+      input.is_a?(Array) ? input : input&.split(",") || []
+    end
+
+    def validate_table_lists(tables, exclude_tables, schema_name)
+      table_list = convert_to_array(tables)
+      exclude_table_list = convert_to_array(exclude_tables)
+
+      if !table_list.empty? && !exclude_table_list.empty?
+        abort_with("Options --tables(-t) and --exclude-tables(-e) cannot be used together.")
+      elsif !table_list.empty?
+        if table_list.size > 0 && (schema_name.nil? || schema_name == "")
+          abort_with("Schema name is required if tables are passed")
+        end
+      elsif exclude_table_list.size > 0 && (schema_name.nil? || schema_name == "")
+        abort_with("Schema name is required if exclude tables are passed")
+      end
+    end
   end
 end

data/lib/pg_easy_replicate/orchestrate.rb

@@ -15,6 +15,7 @@ module PgEasyReplicate
             schema: schema_name,
             conn_string: source_db_url,
             list: options[:tables],
+            exclude_list: options[:exclude_tables],
           )
 
         if options[:recreate_indices_post_copy]
@@ -51,25 +52,19 @@ module PgEasyReplicate
           started_at: Time.now.utc,
           recreate_indices_post_copy: options[:recreate_indices_post_copy],
         )
-      rescue => e
-        stop_sync(
-          group_name: options[:group_name],
-          source_conn_string: source_db_url,
-          target_conn_string: target_db_url,
-        )
 
-        if Group.find(options[:group_name])
-          Group.update(group_name: options[:group_name], failed_at: Time.now)
-        else
-          Group.create(
-            name: options[:group_name],
-            table_names: tables.join(","),
-            schema_name: schema_name,
-            started_at: Time.now.utc,
-            failed_at: Time.now.utc,
+        if options[:track_ddl]
+          DDLManager.setup_ddl_tracking(
+            conn_string: source_db_url,
+            group_name: options[:group_name],
+            schema: schema_name,
           )
         end
-
+      rescue => e
+        stop_sync(group_name: options[:group_name])
+        if Group.find(options[:group_name])
+          Group.update(name: options[:group_name], failed_at: Time.now.utc)
+        end
         abort_with("Starting sync failed: #{e.message}")
       end
 
@@ -179,8 +174,8 @@ module PgEasyReplicate
 
       def stop_sync(
         group_name:,
-        source_conn_string: source_db_url,
-        target_conn_string: target_db_url
+        source_conn_string: nil,
+        target_conn_string: nil
       )
         logger.info(
           "Stopping sync",
@@ -191,29 +186,34 @@ module PgEasyReplicate
         )
         drop_publication(
           group_name: group_name,
-          conn_string: source_conn_string,
+          conn_string: source_conn_string || source_db_url,
         )
         drop_subscription(
           group_name: group_name,
-          target_conn_string: target_conn_string,
+          target_conn_string: target_conn_string || target_db_url,
         )
       rescue => e
-        raise "Unable to stop sync user: #{e.message}"
+        abort_with("Unable to stop sync: #{e.message}")
       end
 
       def switchover(
         group_name:,
-        source_conn_string: source_db_url,
-        target_conn_string: target_db_url,
         lag_delta_size: nil,
-        skip_vacuum_analyze: false
+        skip_vacuum_analyze: false,
+        source_conn_string: nil,
+        target_conn_string: nil
       )
         group = Group.find(group_name)
+        abort_with("Group not found: #{group_name}") unless group
+
         tables_list = group[:table_names].split(",")
 
+        source_conn = source_conn_string || source_db_url
+        target_conn = target_conn_string || target_db_url
+
         unless skip_vacuum_analyze
           run_vacuum_analyze(
-            conn_string: target_conn_string,
+            conn_string: target_conn,
             tables: tables_list,
             schema: group[:schema_name],
           )
@@ -224,39 +224,35 @@ module PgEasyReplicate
         if group[:recreate_indices_post_copy]
           IndexManager.wait_for_replication_completion(group_name: group_name)
           IndexManager.recreate_indices(
-            source_conn_string: source_db_url,
-            target_conn_string: target_db_url,
+            source_conn_string: source_conn,
+            target_conn_string: target_conn,
             tables: tables_list,
             schema: group[:schema_name],
           )
         end
 
-        # Watch for lag again, because it could've grown during index recreation
         watch_lag(group_name: group_name, lag: lag_delta_size || DEFAULT_LAG)
 
         revoke_connections_on_source_db(group_name)
         wait_for_remaining_catchup(group_name)
-        refresh_sequences(
-          conn_string: target_conn_string,
-          schema: group[:schema_name],
-        )
+        refresh_sequences(conn_string: target_conn, schema: group[:schema_name])
         mark_switchover_complete(group_name)
-        # Run vacuum analyze to refresh the planner post switchover
+
         unless skip_vacuum_analyze
           run_vacuum_analyze(
-            conn_string: target_conn_string,
+            conn_string: target_conn,
             tables: tables_list,
             schema: group[:schema_name],
           )
         end
+
         drop_subscription(
           group_name: group_name,
-          target_conn_string: target_conn_string,
+          target_conn_string: target_conn,
         )
       rescue => e
         restore_connections_on_source_db(group_name)
-
-        abort_with("Switchover sync failed: #{e.message}")
+        abort_with("Switchover failed: #{e.message}")
       end
 
       def watch_lag(group_name:, wait_time: DEFAULT_WAIT, lag: DEFAULT_LAG)

data/lib/pg_easy_replicate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEasyReplicate
-  VERSION = "0.2.6"
+  VERSION = "0.3.0"
 end

data/lib/pg_easy_replicate.rb

@@ -6,6 +6,7 @@ require "pg"
 require "sequel"
 require "open3"
 require "English"
+require "pg_query"
 
 require "pg_easy_replicate/helper"
 require "pg_easy_replicate/version"
@@ -15,6 +16,8 @@ require "pg_easy_replicate/orchestrate"
 require "pg_easy_replicate/stats"
 require "pg_easy_replicate/group"
 require "pg_easy_replicate/cli"
+require "pg_easy_replicate/ddl_audit"
+require "pg_easy_replicate/ddl_manager"
 
 Sequel.default_timezone = :utc
 module PgEasyReplicate
@@ -30,11 +33,18 @@ module PgEasyReplicate
       special_user_role: nil,
       copy_schema: false,
       tables: "",
+      exclude_tables: "",
       schema_name: nil
     )
       abort_with("SOURCE_DB_URL is missing") if source_db_url.nil?
       abort_with("TARGET_DB_URL is missing") if target_db_url.nil?
 
+      if !tables.empty? && !exclude_tables.empty?
+        abort_with(
+          "Options --tables(-t) and --exclude-tables(-e) cannot be used together.",
+        )
+      end
+
       system("which pg_dump")
       pg_dump_exists = $CHILD_STATUS.success?
 
@@ -65,6 +75,7 @@ module PgEasyReplicate
               tables_have_replica_identity?(
                 conn_string: source_db_url,
                 tables: tables,
+                exclude_tables: exclude_tables,
                 schema_name: schema_name,
               ),
           }
@@ -77,6 +88,7 @@ module PgEasyReplicate
       special_user_role: nil,
       copy_schema: false,
       tables: "",
+      exclude_tables: "",
       schema_name: nil
     )
       config_hash =
@@ -84,6 +96,7 @@ module PgEasyReplicate
           special_user_role: special_user_role,
           copy_schema: copy_schema,
           tables: tables,
+          exclude_tables: exclude_tables,
           schema_name: schema_name,
         )
 
@@ -103,9 +116,7 @@ module PgEasyReplicate
         abort_with("User on source database does not have super user privilege")
       end
 
-      if tables.split(",").size > 0 && (schema_name.nil? || schema_name == "")
-        abort_with("Schema name is required if tables are passed")
-      end
+      validate_table_lists(tables, exclude_tables, schema_name)
 
       unless config_hash.dig(:tables_have_replica_identity)
         abort_with(
@@ -152,34 +163,66 @@ module PgEasyReplicate
     end
 
     def cleanup(options)
-      logger.info("Dropping groups table")
-      Group.drop
-
-      if options[:everything]
-        logger.info("Dropping schema")
-        drop_internal_schema
-      end
-
-      if options[:everything] || options[:sync]
-        Orchestrate.drop_publication(
-          group_name: options[:group_name],
-          conn_string: source_db_url,
-        )
-
-        Orchestrate.drop_subscription(
-          group_name: options[:group_name],
-          target_conn_string: target_db_url,
+      cleanup_steps = [
+        -> do
+          logger.info("Dropping groups table")
+          Group.drop
+        end,
+        -> do
+          if options[:everything]
+            logger.info("Dropping schema")
+            drop_internal_schema
+          end
+        end,
+        -> do
+          if options[:everything] || options[:sync]
+            logger.info("Dropping publication on source database")
+            Orchestrate.drop_publication(
+              group_name: options[:group_name],
+              conn_string: source_db_url,
+            )
+          end
+        end,
+        -> do
+          if options[:everything] || options[:sync]
+            logger.info("Dropping subscription on target database")
+            Orchestrate.drop_subscription(
+              group_name: options[:group_name],
+              target_conn_string: target_db_url,
+            )
+          end
+        end,
+        -> do
+          if options[:everything]
+            logger.info("Dropping replication user on source database")
+            drop_user(conn_string: source_db_url)
+          end
+        end,
+        -> do
+          if options[:everything]
+            logger.info("Dropping replication user on target database")
+            drop_user(conn_string: target_db_url)
+          end
+          -> do
+            if options[:everything]
+              PgEasyReplicate::DDLManager.cleanup_ddl_tracking(
+                conn_string: source_db_url,
+                group_name: options[:group_name],
+              )
+            end
+          end
+        end,
+      ]
+
+      cleanup_steps.each do |step|
+        step.call
+      rescue => e
+        logger.warn(
+          "Part of the cleanup step failed with #{e.message}. Continuing...",
         )
       end
 
-      if options[:everything]
-        # Drop users at last
-        logger.info("Dropping replication user on source database")
-        drop_user(conn_string: source_db_url)
-
-        logger.info("Dropping replication user on target database")
-        drop_user(conn_string: target_db_url)
-      end
+      logger.info("Cleanup process completed.")
     rescue => e
       abort_with("Unable to cleanup: #{e.message}")
     end
@@ -299,9 +342,9 @@ module PgEasyReplicate
       special_user_role: nil,
       grant_permissions_on_schema: false
     )
-      password = connection_info(conn_string)[:password].gsub("'") { "''" }
+      return if user_exists?(conn_string: conn_string, user: internal_user_name)
 
-      drop_user(conn_string: conn_string)
+      password = connection_info(conn_string)[:password].gsub("'") { "''" }
 
       sql = <<~SQL
         create role #{quote_ident(internal_user_name)} with password '#{password}' login createdb createrole;
@@ -390,6 +433,7 @@ module PgEasyReplicate
     def tables_have_replica_identity?(
       conn_string:,
       tables: "",
+      exclude_tables: "",
       schema_name: nil
     )
       schema_name ||= "public"
@@ -399,6 +443,7 @@ module PgEasyReplicate
           schema: schema_name,
           conn_string: source_db_url,
           list: tables,
+          exclude_list: exclude_tables,
         )
       return false if table_list.empty?
 

data/scripts/e2e-start.sh

@@ -10,9 +10,17 @@ export SOURCE_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhos
 export TARGET_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhost:5433/postgres-db"
 export PGPASSWORD='james-bond123@7!'"'"''"'"'3aaR'
 
-# Config check, Bootstrap and cleanup
 echo "===== Performing Bootstrap and cleanup"
 bundle exec bin/pg_easy_replicate bootstrap -g cluster-1 --copy-schema
-bundle exec bin/pg_easy_replicate start_sync -g cluster-1 -s public --recreate-indices-post-copy
+bundle exec bin/pg_easy_replicate start_sync -g cluster-1 -s public --recreate-indices-post-copy --track-ddl
 bundle exec bin/pg_easy_replicate stats -g cluster-1
+
+echo "===== Applying DDL change"
+psql $SOURCE_DB_URL -c "ALTER TABLE public.pgbench_accounts ADD COLUMN test_column VARCHAR(255)"
+
+echo "===== Applying DDL changes"
+echo "Y" | bundle exec bin/pg_easy_replicate apply_ddl_change -g cluster-1
+
+# Switchover
+echo "===== Performing switchover"
 bundle exec bin/pg_easy_replicate switchover -g cluster-1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_easy_replicate
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.3.0
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-04 00:00:00.000000000 Z
+date: 2024-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.5.3
+- !ruby/object:Gem::Dependency
+  name: pg_query
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.0
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -47,7 +61,7 @@ dependencies:
         version: '5.69'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.82'
+        version: '5.83'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -57,7 +71,7 @@ dependencies:
         version: '5.69'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.82'
+        version: '5.83'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -274,6 +288,8 @@ files:
 - docker-compose.yml
 - lib/pg_easy_replicate.rb
 - lib/pg_easy_replicate/cli.rb
+- lib/pg_easy_replicate/ddl_audit.rb
+- lib/pg_easy_replicate/ddl_manager.rb
 - lib/pg_easy_replicate/group.rb
 - lib/pg_easy_replicate/helper.rb
 - lib/pg_easy_replicate/index_manager.rb