pg_easy_replicate 0.2.1 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a39a74d7e35cec6dce951d23eff51972e22811779ab8637edfa989567e62620
-  data.tar.gz: 6a81b795c7968b5d5c218f35084cdf40bf558271fc8e290ef75fe28a570ce9b6
+  metadata.gz: 86261e2b60e55c5770bbd1537d5a3a501a514d1eaf8e741c5c81890d864b0a0f
+  data.tar.gz: bf6a710930b4f9533d01c3f66e3b91c02ad5a422df9d17be2bbaf72cdb979a2e
 SHA512:
-  metadata.gz: f9af65ec14a25975af9fab3efa884d91aa90a43aae74b355370608b3c5e61453f6fd71dd333bd8917193f8b90ec86965464893f9aae2f1a965f2b40eaeebc882
-  data.tar.gz: 9643dc3de2102b82ab29d085e90b3dfbbfa923d0c3324d91e1bff8fa7082a8136263e59566559cabbf230a3094648a2a52152912b24dd760ffc3c727b5a11f39
+  metadata.gz: 5d7aab6e89948a45248fcb19fb6d3e9a6982adbc23cba094fe413b2a376cbffb1bbfdb011833f16c4c3aeb3c4d3e1192792699b4c360a20555fc05972e2ae111
+  data.tar.gz: a4c72947b466664dfc716a2ed21cab159d0f6768b3ae74c9c95db105bafb22da57244236a1f9677e1936af8140989dae9a0c5b1729c2f4bc62dfcbbad0d6a007

data/CHANGELOG.md

@@ -1,4 +1,8 @@
-## [0.2.1] - 2023-12-29
+## [0.2.1] - 2024-01-21
+
+- Extend config check to assert for REPLICA IDENTITY on tables and drop index bug - #88
+
+## [0.2.1] - 2024-01-20
 
 - Don't attempt to drop and recreate unique indices - #88
 - Dependency updates

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pg_easy_replicate (0.2.1)
+    pg_easy_replicate (0.2.3)
       ougai (~> 2.0.0)
       pg (~> 1.5.3)
       sequel (>= 5.69, < 5.77)

data/README.md

@@ -4,7 +4,7 @@
 [![Smoke spec](https://github.com/shayonj/pg_easy_replicate/actions/workflows/smoke.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg_easy_replicate/actions/workflows/ci.yaml)
 [![Gem Version](https://badge.fury.io/rb/pg_easy_replicate.svg?2)](https://badge.fury.io/rb/pg_easy_replicate)
 
-`pg_easy_replicate` is a CLI orchestrator tool that simplifies the process of setting up [logical replication](https://www.postgresql.org/docs/current/logical-replication.html) between two PostgreSQL databases. `pg_easy_replicate` also supports switchover. After the source (primary database) is fully replicated, `pg_easy_replicate` puts it into read-only mode and via logical replication flushes all data to the new target database. This ensures zero data loss and minimal downtime for the application. This method can be useful for performing minimal downtime (up to <1min, depending) major version upgrades between two PostgreSQL databases, load testing with blue/green database setup and other similar use cases.
+`pg_easy_replicate` is a CLI orchestrator tool that simplifies the process of setting up [logical replication](https://www.postgresql.org/docs/current/logical-replication.html) between two PostgreSQL databases. `pg_easy_replicate` also supports switchover. After the source (primary database) is fully replicated, `pg_easy_replicate` puts it into read-only mode and via logical replication flushes all data to the new target database. This ensures zero data loss and minimal downtime for the application. This method can be useful for performing minimal downtime (up to <1min, depending) major version upgrades between a Blue/Green PostgreSQL database setup, load testing and other similar use cases.
 
 Battle tested in production at [Tines](https://www.tines.com/) 🚀
 
@@ -19,7 +19,7 @@ Battle tested in production at [Tines](https://www.tines.com/) 🚀
   - [Config check](#config-check)
   - [Bootstrap](#bootstrap)
   - [Bootstrap and Config Check with special user role in AWS or GCP](#bootstrap-and-config-check-with-special-user-role-in-aws-or-gcp)
-    - [Config Check](#config-check)
+    - [Config Check](#config-check-1)
     - [Bootstrap](#bootstrap-1)
   - [Start sync](#start-sync)
   - [Stats](#stats)
@@ -29,7 +29,7 @@ Battle tested in production at [Tines](https://www.tines.com/) 🚀
   - [Rolling restart strategy](#rolling-restart-strategy)
   - [DNS Failover strategy](#dns-failover-strategy)
 - [FAQ](#faq)
-  - [Adding internal user to pgBouncer `userlist`](#adding-internal-user-to-pgbouncer-userlist)
+  - [Adding internal user to `pg_hba` or pgBouncer `userlist`](#adding-internal-user-to-pg_hba-or-pgbouncer-userlist)
 - [Contributing](#contributing)
 
 ## Installation
@@ -61,6 +61,7 @@ https://hub.docker.com/r/shayonj/pg_easy_replicate
 - PostgreSQL 10 and later
 - Ruby 3.0 and later
 - Database users should have `SUPERUSER` permissions, or pass in a special user with privileges to create the needed role, schema, publication and subscription on both databases. More on `--special-user-role` section below.
+- See more on [FAQ](#faq) below
 
 ## Limits
 
@@ -256,9 +257,9 @@ Next, you can set up a program that watches the `stats` and waits until `switcho
 
 ## FAQ
 
-### Adding internal user to pgBouncer `userlist`
+### Adding internal user to `pg_hba` or pgBouncer `userlist`
 
-`pg_easy_replicate` creates a special user to orchestrate the replication. If you use pgBouncer, you may need to allow `pger_su_h1a4fb` as a user that can perform login by adding it to the `userlist`.
+`pg_easy_replicate` sets up a designated user for managing the replication process. In case you handle user permissions through `pg_hba`, it's necessary to modify this list to permit sessions from `pger_su_h1a4fb`. Similarly, with pgBouncer, you'll need to authorize `pger_su_h1a4fb` for login access by including it in the `userlist`.
 
 ## Contributing
 

data/lib/pg_easy_replicate/cli.rb

@@ -16,10 +16,21 @@ module PgEasyReplicate
                   aliases: "-c",
                   boolean: true,
                   desc: "Copy schema to the new database"
+    method_option :tables,
+                  aliases: "-t",
+                  default: "",
+                  desc:
+                    "Comma separated list of table names. Default: All tables"
+    method_option :schema_name,
+                  aliases: "-s",
+                  desc:
+                    "Name of the schema tables are in, only required if passing list of tables"
     def config_check
       PgEasyReplicate.assert_config(
         special_user_role: options[:special_user_role],
         copy_schema: options[:copy_schema],
+        tables: options[:tables],
+        schema_name: options[:schema_name],
       )
 
       puts "✅ Config is looking good."
@@ -77,6 +88,7 @@ module PgEasyReplicate
                     "Name of the schema tables are in, only required if passing list of tables"
     method_option :tables,
                   aliases: "-t",
+                  default: "",
                   desc:
                     "Comma separated list of table names. Default: All tables"
     method_option :recreate_indices_post_copy,
@@ -107,8 +119,12 @@ module PgEasyReplicate
                   desc: "Name of the group previously provisioned"
     method_option :lag_delta_size,
                   aliases: "-l",
-                  desc:
-                    "The size of the lag to watch for before switchover. Default 200KB."
+                  desc: "The size of the lag to watch for before switchover. Default 200KB."
+    method_option :skip_vacuum_analyze,
+                  type: :boolean,
+                  default: false,
+                  aliases: "-s",
+                  desc: "Skip vacuum analyzing tables before switchover."
     # method_option :bi_directional,
     #               aliases: "-b",
     #               desc:
@@ -117,6 +133,7 @@ module PgEasyReplicate
       PgEasyReplicate::Orchestrate.switchover(
         group_name: options[:group_name],
         lag_delta_size: options[:lag_delta_size],
+        skip_vacuum_analyze: options[:skip_vacuum_analyze]
       )
     end
 

data/lib/pg_easy_replicate/helper.rb

@@ -72,5 +72,32 @@ module PgEasyReplicate
       raise(msg) if test_env?
       abort(msg)
     end
+
+    def determine_tables(conn_string:, list: "", schema: nil)
+      schema ||= "public"
+
+      tables = list&.split(",") || []
+      if tables.size > 0
+        tables
+      else
+        list_all_tables(schema: schema, conn_string: conn_string)
+      end
+    end
+
+    def list_all_tables(schema:, conn_string:)
+      Query
+        .run(
+          query:
+            "SELECT table_name
+             FROM information_schema.tables
+             WHERE table_schema = '#{schema}' AND
+               table_type = 'BASE TABLE'
+             ORDER BY table_name",
+          connection_url: conn_string,
+          user: db_user(conn_string),
+        )
+        .map(&:values)
+        .flatten
+    end
   end
 end

data/lib/pg_easy_replicate/index_manager.rb

@@ -17,7 +17,7 @@ module PgEasyReplicate
         tables: tables,
         schema: schema,
       ).each do |index|
-        drop_sql = "DROP INDEX CONCURRENTLY #{index[:index_name]};"
+        drop_sql = "DROP INDEX CONCURRENTLY #{schema}.#{index[:index_name]};"
 
         Query.run(
           query: drop_sql,
@@ -56,8 +56,8 @@ module PgEasyReplicate
     end
 
     def self.fetch_indices(conn_string:, tables:, schema:)
-      return [] if tables.split(",").empty?
-      table_list = tables.split(",").map { |table| "'#{table}'" }.join(",")
+      return [] if tables.empty?
+      table_list = tables.map { |table| "'#{table}'" }.join(",")
 
       sql = <<-SQL
         SELECT

data/lib/pg_easy_replicate/orchestrate.rb

@@ -46,7 +46,7 @@ module PgEasyReplicate
 
         Group.create(
           name: options[:group_name],
-          table_names: tables,
+          table_names: tables.join(","),
           schema_name: schema_name,
           started_at: Time.now.utc,
           recreate_indices_post_copy: options[:recreate_indices_post_copy],
@@ -63,7 +63,7 @@ module PgEasyReplicate
         else
           Group.create(
             name: options[:group_name],
-            table_names: tables,
+            table_names: tables.join(","),
             schema_name: schema_name,
             started_at: Time.now.utc,
             failed_at: Time.now.utc,
@@ -92,42 +92,24 @@ module PgEasyReplicate
         schema:,
         group_name:,
         conn_string:,
-        tables: ""
+        tables: []
       )
         logger.info(
           "Adding tables up publication",
           { publication_name: publication_name(group_name) },
         )
 
-        tables
-          .split(",")
-          .map do |table_name|
-            Query.run(
-              query:
-                "ALTER PUBLICATION #{quote_ident(publication_name(group_name))}
-                        ADD TABLE #{quote_ident(table_name)}",
-              connection_url: conn_string,
-              schema: schema,
-            )
-          end
-      rescue => e
-        raise "Unable to add tables to publication: #{e.message}"
-      end
-
-      def list_all_tables(schema:, conn_string:)
-        Query
-          .run(
+        tables.map do |table_name|
+          Query.run(
             query:
-              "SELECT table_name
-               FROM information_schema.tables
-               WHERE table_schema = '#{schema}' AND
-                 table_type = 'BASE TABLE'
-               ORDER BY table_name",
+              "ALTER PUBLICATION #{quote_ident(publication_name(group_name))}
+                        ADD TABLE #{quote_ident(table_name)}",
             connection_url: conn_string,
+            schema: schema,
           )
-          .map(&:values)
-          .flatten
-          .join(",")
+        end
+      rescue => e
+        raise "Unable to add tables to publication: #{e.message}"
       end
 
       def drop_publication(group_name:, conn_string:)
@@ -222,15 +204,19 @@ module PgEasyReplicate
         group_name:,
         source_conn_string: source_db_url,
         target_conn_string: target_db_url,
-        lag_delta_size: nil
+        lag_delta_size: nil,
+        skip_vacuum_analyze: false
       )
         group = Group.find(group_name)
+        tables_list = group[:table_names].split(",")
 
-        run_vacuum_analyze(
-          conn_string: target_conn_string,
-          tables: group[:table_names],
-          schema: group[:schema_name],
-        )
+        unless skip_vacuum_analyze
+          run_vacuum_analyze(
+            conn_string: target_conn_string,
+            tables: tables_list,
+            schema: group[:schema_name],
+          )
+        end
 
         watch_lag(group_name: group_name, lag: lag_delta_size || DEFAULT_LAG)
 
@@ -239,7 +225,7 @@ module PgEasyReplicate
           IndexManager.recreate_indices(
             source_conn_string: source_db_url,
             target_conn_string: target_db_url,
-            tables: group[:table_names],
+            tables: tables_list,
             schema: group[:schema_name],
           )
         end
@@ -255,11 +241,13 @@ module PgEasyReplicate
         )
         mark_switchover_complete(group_name)
         # Run vacuum analyze to refresh the planner post switchover
-        run_vacuum_analyze(
-          conn_string: target_conn_string,
-          tables: group[:table_names],
-          schema: group[:schema_name],
-        )
+        unless skip_vacuum_analyze
+          run_vacuum_analyze(
+            conn_string: target_conn_string,
+            tables: tables_list,
+            schema: group[:schema_name],
+          )
+        end
         drop_subscription(
           group_name: group_name,
           target_conn_string: target_conn_string,
@@ -369,21 +357,21 @@ module PgEasyReplicate
       end
 
       def run_vacuum_analyze(conn_string:, tables:, schema:)
-        tables
-          .split(",")
-          .each do |t|
-            logger.info(
-              "Running vacuum analyze on #{t}",
-              schema: schema,
-              table: t,
-            )
-            Query.run(
-              query: "VACUUM VERBOSE ANALYZE #{t}",
-              connection_url: conn_string,
-              schema: schema,
-              transaction: false,
-            )
-          end
+        tables.each do |t|
+          logger.info(
+            "Running vacuum analyze on #{t}",
+            schema: schema,
+            table: t,
+          )
+
+          Query.run(
+            query: "VACUUM VERBOSE ANALYZE #{t};",
+            connection_url: conn_string,
+            schema: schema,
+            transaction: false,
+            using_vacuum_analyze: true,
+          )
+        end
       rescue => e
         raise "Unable to run vacuum and analyze: #{e.message}"
       end
@@ -391,16 +379,6 @@ module PgEasyReplicate
       def mark_switchover_complete(group_name)
         Group.update(group_name: group_name, switchover_completed_at: Time.now)
       end
-
-      private
-
-      def determine_tables(schema:, conn_string:, list: "")
-        tables = list&.split(",") || []
-        unless tables.size > 0
-          return list_all_tables(schema: schema, conn_string: conn_string)
-        end
-        ""
-      end
     end
   end
 end

data/lib/pg_easy_replicate/query.rb

@@ -10,20 +10,26 @@ module PgEasyReplicate
         connection_url:,
         user: internal_user_name,
         schema: nil,
-        transaction: true
+        transaction: true,
+        using_vacuum_analyze: false
       )
         conn =
           connect(connection_url: connection_url, schema: schema, user: user)
+        timeout ||= "5s"
         if transaction
           r =
             conn.transaction do
               conn.run("SET search_path to #{quote_ident(schema)}") if schema
-              conn.run("SET statement_timeout to '5s'")
+              conn.run("SET statement_timeout to '#{timeout}'")
               conn.fetch(query).to_a
             end
         else
           conn.run("SET search_path to #{quote_ident(schema)}") if schema
-          conn.run("SET statement_timeout to '5s'")
+          if using_vacuum_analyze
+            conn.run("SET statement_timeout=0")
+          else
+            conn.run("SET statement_timeout to '5s'")
+          end
           r = conn.fetch(query).to_a
         end
         conn.disconnect

data/lib/pg_easy_replicate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEasyReplicate
-  VERSION = "0.2.1"
+  VERSION = "0.2.3"
 end

data/lib/pg_easy_replicate.rb

@@ -26,7 +26,12 @@ module PgEasyReplicate
   extend Helper
 
   class << self
-    def config(special_user_role: nil, copy_schema: false)
+    def config(
+      special_user_role: nil,
+      copy_schema: false,
+      tables: "",
+      schema_name: nil
+    )
       abort_with("SOURCE_DB_URL is missing") if source_db_url.nil?
       abort_with("TARGET_DB_URL is missing") if target_db_url.nil?
 
@@ -56,15 +61,31 @@ module PgEasyReplicate
                 user: db_user(target_db_url),
               ),
             pg_dump_exists: pg_dump_exists,
+            tables_have_replica_identity:
+              tables_have_replica_identity?(
+                conn_string: source_db_url,
+                tables: tables,
+                schema_name: schema_name,
+              ),
           }
         rescue => e
           abort_with("Unable to check config: #{e.message}")
         end
     end
 
-    def assert_config(special_user_role: nil, copy_schema: false)
+    def assert_config(
+      special_user_role: nil,
+      copy_schema: false,
+      tables: "",
+      schema_name: nil
+    )
       config_hash =
-        config(special_user_role: special_user_role, copy_schema: copy_schema)
+        config(
+          special_user_role: special_user_role,
+          copy_schema: copy_schema,
+          tables: tables,
+          schema_name: schema_name,
+        )
 
       if copy_schema && !config_hash.dig(:pg_dump_exists)
         abort_with("pg_dump must exist if copy_schema (-c) is passed")
@@ -82,6 +103,19 @@ module PgEasyReplicate
         abort_with("User on source database does not have super user privilege")
       end
 
+      if tables.split(",").size > 0 && (schema_name.nil? || schema_name == "")
+        abort_with("Schema name is required if tables are passed")
+      end
+
+      unless config_hash.dig(:tables_have_replica_identity)
+        abort_with(
+          "Ensure all tables involved in logical replication have an appropriate replica identity set. This can be done using:
+        1. Default (Primary Key): `ALTER TABLE table_name REPLICA IDENTITY DEFAULT;`
+        2. Unique Index: `ALTER TABLE table_name REPLICA IDENTITY USING INDEX index_name;`
+        3. Full (All Columns): `ALTER TABLE table_name REPLICA IDENTITY FULL;`",
+        )
+      end
+
       return if config_hash.dig(:target_db_is_super_user)
       abort_with("User on target database does not have super user privilege")
     end
@@ -352,5 +386,47 @@ module PgEasyReplicate
         )
         .any? { |q| q[:username] == user }
     end
+
+    def tables_have_replica_identity?(
+      conn_string:,
+      tables: "",
+      schema_name: nil
+    )
+      schema_name ||= "public"
+
+      table_list =
+        determine_tables(
+          schema: schema_name,
+          conn_string: source_db_url,
+          list: tables,
+        )
+      return false if table_list.empty?
+
+      formatted_table_list = table_list.map { |table| "'#{table}'" }.join(", ")
+
+      sql = <<~SQL
+        SELECT t.relname AS table_name,
+              CASE
+                WHEN t.relreplident = 'd' THEN 'default'
+                WHEN t.relreplident = 'n' THEN 'nothing'
+                WHEN t.relreplident = 'i' THEN 'index'
+                WHEN t.relreplident = 'f' THEN 'full'
+              END AS replica_identity
+        FROM pg_class t
+        JOIN pg_namespace ns ON t.relnamespace = ns.oid
+        WHERE ns.nspname = '#{schema_name}'
+          AND t.relkind = 'r'
+          AND t.relname IN (#{formatted_table_list})
+      SQL
+
+      results =
+        Query.run(
+          query: sql,
+          connection_url: conn_string,
+          user: db_user(conn_string),
+        )
+
+      results.all? { |r| r[:replica_identity] != "nothing" }
+    end
   end
 end

data/scripts/e2e-start.sh

@@ -10,7 +10,7 @@ export SOURCE_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhos
 export TARGET_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhost:5433/postgres-db"
 export PGPASSWORD='james-bond123@7!'"'"''"'"'3aaR'
 
-# Bootstrap and cleanup
+# Config check, Bootstrap and cleanup
 echo "===== Performing Bootstrap and cleanup"
 bundle exec bin/pg_easy_replicate bootstrap -g cluster-1 --copy-schema
 bundle exec bin/pg_easy_replicate start_sync -g cluster-1 -s public --recreate-indices-post-copy

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_easy_replicate
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.3
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-20 00:00:00.000000000 Z
+date: 2024-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai