pg_easy_replicate 0.2.0 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da825eb796e5b827c9197440301d841f725a5bbaeac74b066fea67b5bbf039ac
-  data.tar.gz: 68da98e57eab118b7aea8f4e83171eaa8b18bd66a86bd5f39fb901f5aef9bfda
+  metadata.gz: efcbf369a4410abd783ea1dba49ead97fa471705842df4a35d28864349b1c70c
+  data.tar.gz: 676d97aa0c551dd0f9d58c3bccb229951bcb2dfc34653995375c359718bc1d7f
 SHA512:
-  metadata.gz: e04ca73e6f6b63f68a37389cb788bc7a6fddd80aa75fa433d297d4a600b1929ee06341e456b8b2d12a0cb03952824c02dca63033f40675948c250bcba2849b2a
-  data.tar.gz: b0617c3d5d2defeab8f576eb6cceb1845c8c7fe7478a064f2109fbd4e63ebcf58867bd2557221810c53778e62806bacd65e92470fd368f96c5d38405bee4efec
+  metadata.gz: 774ef530f4be4b409fdf2d4956a44deb09aebf59c8ef480074d426ca4c72006673c7a6915d860e61846fb00df8e14fa9534e920f5802873ed727e37564197b25
+  data.tar.gz: 50055941c3a83351b22ceaeb539b807a7cec5e3130732944c63e8a5440a7707a273a546681b1d581d4f79d9fba616998fc48cffc009e401bfc570e76e27064f0

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## [0.2.1] - 2023-12-29
+
+- Don't attempt to drop and recreate unique indices - #88
+- Dependency updates
+
+## [0.2.0] - 2023-12-29
+
+- Recreate indices post COPY, once all tables are in replicating mode - #81
+
 ## [0.1.12] - 2023-12-13
 
 - Bump rubocop-rspec from 2.24.1 to 2.25.0 - #65

data/Gemfile.lock

@@ -1,31 +1,31 @@
 PATH
   remote: .
   specs:
-    pg_easy_replicate (0.2.0)
+    pg_easy_replicate (0.2.2)
       ougai (~> 2.0.0)
       pg (~> 1.5.3)
-      sequel (>= 5.69, < 5.76)
+      sequel (>= 5.69, < 5.77)
       thor (>= 1.2.2, < 1.4.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
-    bigdecimal (3.1.4)
+    bigdecimal (3.1.5)
     coderay (1.1.3)
     diff-lcs (1.5.0)
     haml (6.1.1)
       temple (>= 0.8.2)
       thor
       tilt
-    json (2.7.0)
+    json (2.7.1)
     language_server-protocol (3.17.0.3)
     method_source (1.0.0)
     oj (3.14.3)
     ougai (2.0.0)
       oj (~> 3.10)
-    parallel (1.23.0)
-    parser (3.2.2.4)
+    parallel (1.24.0)
+    parser (3.3.0.4)
       ast (~> 2.4.1)
       racc
     pg (1.5.4)
@@ -37,7 +37,7 @@ GEM
     rainbow (3.1.1)
     rake (13.1.0)
     rbs (3.1.0)
-    regexp_parser (2.8.2)
+    regexp_parser (2.9.0)
     rexml (3.2.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -52,11 +52,11 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    rubocop (1.58.0)
+    rubocop (1.60.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.2.4)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
@@ -65,23 +65,23 @@ GEM
       unicode-display_width (>= 2.4.0, < 3.0)
     rubocop-ast (1.30.0)
       parser (>= 3.2.1.0)
-    rubocop-capybara (2.19.0)
+    rubocop-capybara (2.20.0)
+      rubocop (~> 1.41)
+    rubocop-factory_bot (2.25.1)
       rubocop (~> 1.41)
-    rubocop-factory_bot (2.24.0)
-      rubocop (~> 1.33)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-    rubocop-performance (1.19.1)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
+    rubocop-performance (1.20.2)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
-    rubocop-rspec (2.25.0)
+    rubocop-rspec (2.26.1)
       rubocop (~> 1.40)
       rubocop-capybara (~> 2.17)
       rubocop-factory_bot (~> 2.22)
     ruby-progressbar (1.13.0)
-    sequel (5.75.0)
+    sequel (5.76.0)
       bigdecimal
     syntax_tree (6.2.0)
       prettier_print (>= 1.2.0)

data/lib/pg_easy_replicate/cli.rb

@@ -16,10 +16,20 @@ module PgEasyReplicate
                   aliases: "-c",
                   boolean: true,
                   desc: "Copy schema to the new database"
+    method_option :tables,
+                  aliases: "-t",
+                  desc:
+                    "Comma separated list of table names. Default: All tables"
+    method_option :schema_name,
+                  aliases: "-s",
+                  desc:
+                    "Name of the schema tables are in, only required if passing list of tables"
     def config_check
       PgEasyReplicate.assert_config(
         special_user_role: options[:special_user_role],
         copy_schema: options[:copy_schema],
+        tables: options[:tables],
+        schema_name: options[:schema_name],
       )
 
       puts "✅ Config is looking good."

data/lib/pg_easy_replicate/helper.rb

@@ -72,5 +72,32 @@ module PgEasyReplicate
       raise(msg) if test_env?
       abort(msg)
     end
+
+    def determine_tables(conn_string:, list: "", schema: nil)
+      schema ||= "public"
+
+      tables = list&.split(",") || []
+      if tables.size > 0
+        tables
+      else
+        list_all_tables(schema: schema, conn_string: conn_string)
+      end
+    end
+
+    def list_all_tables(schema:, conn_string:)
+      Query
+        .run(
+          query:
+            "SELECT table_name
+             FROM information_schema.tables
+             WHERE table_schema = '#{schema}' AND
+               table_type = 'BASE TABLE'
+             ORDER BY table_name",
+          connection_url: conn_string,
+          user: db_user(conn_string),
+        )
+        .map(&:values)
+        .flatten
+    end
   end
 end

data/lib/pg_easy_replicate/index_manager.rb

@@ -17,7 +17,7 @@ module PgEasyReplicate
         tables: tables,
         schema: schema,
       ).each do |index|
-        drop_sql = "DROP INDEX CONCURRENTLY #{index[:index_name]};"
+        drop_sql = "DROP INDEX CONCURRENTLY #{schema}.#{index[:index_name]};"
 
         Query.run(
           query: drop_sql,
@@ -56,8 +56,8 @@ module PgEasyReplicate
     end
 
     def self.fetch_indices(conn_string:, tables:, schema:)
-      return [] if tables.split(",").empty?
-      table_list = tables.split(",").map { |table| "'#{table}'" }.join(",")
+      return [] if tables.empty?
+      table_list = tables.map { |table| "'#{table}'" }.join(",")
 
       sql = <<-SQL
         SELECT
@@ -75,6 +75,7 @@ module PgEasyReplicate
             AND n.oid = t.relnamespace
             AND t.relkind = 'r'  -- only find indexes of tables
             AND ix.indisprimary = FALSE  -- exclude primary keys
+            AND ix.indisunique = FALSE  -- exclude unique indexes
             AND n.nspname = '#{schema}'
             AND t.relname IN (#{table_list})
         ORDER BY

data/lib/pg_easy_replicate/orchestrate.rb

@@ -46,7 +46,7 @@ module PgEasyReplicate
 
         Group.create(
           name: options[:group_name],
-          table_names: tables,
+          table_names: tables.join(","),
           schema_name: schema_name,
           started_at: Time.now.utc,
           recreate_indices_post_copy: options[:recreate_indices_post_copy],
@@ -63,7 +63,7 @@ module PgEasyReplicate
         else
           Group.create(
             name: options[:group_name],
-            table_names: tables,
+            table_names: tables.join(","),
             schema_name: schema_name,
             started_at: Time.now.utc,
             failed_at: Time.now.utc,
@@ -92,42 +92,24 @@ module PgEasyReplicate
         schema:,
         group_name:,
         conn_string:,
-        tables: ""
+        tables: []
       )
         logger.info(
           "Adding tables up publication",
           { publication_name: publication_name(group_name) },
         )
 
-        tables
-          .split(",")
-          .map do |table_name|
-            Query.run(
-              query:
-                "ALTER PUBLICATION #{quote_ident(publication_name(group_name))}
-                        ADD TABLE #{quote_ident(table_name)}",
-              connection_url: conn_string,
-              schema: schema,
-            )
-          end
-      rescue => e
-        raise "Unable to add tables to publication: #{e.message}"
-      end
-
-      def list_all_tables(schema:, conn_string:)
-        Query
-          .run(
+        tables.map do |table_name|
+          Query.run(
             query:
-              "SELECT table_name
-               FROM information_schema.tables
-               WHERE table_schema = '#{schema}' AND
-                 table_type = 'BASE TABLE'
-               ORDER BY table_name",
+              "ALTER PUBLICATION #{quote_ident(publication_name(group_name))}
+                        ADD TABLE #{quote_ident(table_name)}",
             connection_url: conn_string,
+            schema: schema,
           )
-          .map(&:values)
-          .flatten
-          .join(",")
+        end
+      rescue => e
+        raise "Unable to add tables to publication: #{e.message}"
       end
 
       def drop_publication(group_name:, conn_string:)
@@ -225,10 +207,11 @@ module PgEasyReplicate
         lag_delta_size: nil
       )
         group = Group.find(group_name)
+        tables_list = group[:table_names].split(",")
 
         run_vacuum_analyze(
           conn_string: target_conn_string,
-          tables: group[:table_names],
+          tables: tables_list,
           schema: group[:schema_name],
         )
 
@@ -239,7 +222,7 @@ module PgEasyReplicate
           IndexManager.recreate_indices(
             source_conn_string: source_db_url,
             target_conn_string: target_db_url,
-            tables: group[:table_names],
+            tables: tables_list,
             schema: group[:schema_name],
           )
         end
@@ -257,7 +240,7 @@ module PgEasyReplicate
         # Run vacuum analyze to refresh the planner post switchover
         run_vacuum_analyze(
           conn_string: target_conn_string,
-          tables: group[:table_names],
+          tables: tables_list,
           schema: group[:schema_name],
         )
         drop_subscription(
@@ -369,21 +352,19 @@ module PgEasyReplicate
       end
 
       def run_vacuum_analyze(conn_string:, tables:, schema:)
-        tables
-          .split(",")
-          .each do |t|
-            logger.info(
-              "Running vacuum analyze on #{t}",
-              schema: schema,
-              table: t,
-            )
-            Query.run(
-              query: "VACUUM VERBOSE ANALYZE #{t}",
-              connection_url: conn_string,
-              schema: schema,
-              transaction: false,
-            )
-          end
+        tables.each do |t|
+          logger.info(
+            "Running vacuum analyze on #{t}",
+            schema: schema,
+            table: t,
+          )
+          Query.run(
+            query: "VACUUM VERBOSE ANALYZE #{t}",
+            connection_url: conn_string,
+            schema: schema,
+            transaction: false,
+          )
+        end
       rescue => e
         raise "Unable to run vacuum and analyze: #{e.message}"
       end
@@ -391,16 +372,6 @@ module PgEasyReplicate
       def mark_switchover_complete(group_name)
         Group.update(group_name: group_name, switchover_completed_at: Time.now)
       end
-
-      private
-
-      def determine_tables(schema:, conn_string:, list: "")
-        tables = list&.split(",") || []
-        unless tables.size > 0
-          return list_all_tables(schema: schema, conn_string: conn_string)
-        end
-        ""
-      end
     end
   end
 end

data/lib/pg_easy_replicate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEasyReplicate
-  VERSION = "0.2.0"
+  VERSION = "0.2.2"
 end

data/lib/pg_easy_replicate.rb

@@ -26,7 +26,12 @@ module PgEasyReplicate
   extend Helper
 
   class << self
-    def config(special_user_role: nil, copy_schema: false)
+    def config(
+      special_user_role: nil,
+      copy_schema: false,
+      tables: "",
+      schema_name: nil
+    )
       abort_with("SOURCE_DB_URL is missing") if source_db_url.nil?
       abort_with("TARGET_DB_URL is missing") if target_db_url.nil?
 
@@ -56,15 +61,31 @@ module PgEasyReplicate
                 user: db_user(target_db_url),
               ),
             pg_dump_exists: pg_dump_exists,
+            tables_have_replica_identity:
+              tables_have_replica_identity?(
+                conn_string: source_db_url,
+                tables: tables,
+                schema_name: schema_name,
+              ),
           }
         rescue => e
           abort_with("Unable to check config: #{e.message}")
         end
     end
 
-    def assert_config(special_user_role: nil, copy_schema: false)
+    def assert_config(
+      special_user_role: nil,
+      copy_schema: false,
+      tables: "",
+      schema_name: nil
+    )
       config_hash =
-        config(special_user_role: special_user_role, copy_schema: copy_schema)
+        config(
+          special_user_role: special_user_role,
+          copy_schema: copy_schema,
+          tables: tables,
+          schema_name: schema_name,
+        )
 
       if copy_schema && !config_hash.dig(:pg_dump_exists)
         abort_with("pg_dump must exist if copy_schema (-c) is passed")
@@ -82,6 +103,19 @@ module PgEasyReplicate
         abort_with("User on source database does not have super user privilege")
       end
 
+      if tables.split(",").size > 0 && (schema_name.nil? || schema_name == "")
+        abort_with("Schema name is required if tables are passed")
+      end
+
+      unless config_hash.dig(:tables_have_replica_identity)
+        abort_with(
+          "Ensure all tables involved in logical replication have an appropriate replica identity set. This can be done using:
+        1. Default (Primary Key): `ALTER TABLE table_name REPLICA IDENTITY DEFAULT;`
+        2. Unique Index: `ALTER TABLE table_name REPLICA IDENTITY USING INDEX index_name;`
+        3. Full (All Columns): `ALTER TABLE table_name REPLICA IDENTITY FULL;`",
+        )
+      end
+
       return if config_hash.dig(:target_db_is_super_user)
       abort_with("User on target database does not have super user privilege")
     end
@@ -352,5 +386,47 @@ module PgEasyReplicate
         )
         .any? { |q| q[:username] == user }
     end
+
+    def tables_have_replica_identity?(
+      conn_string:,
+      tables: "",
+      schema_name: nil
+    )
+      schema_name ||= "public"
+
+      table_list =
+        determine_tables(
+          schema: schema_name,
+          conn_string: source_db_url,
+          list: tables,
+        )
+      return false if table_list.empty?
+
+      formatted_table_list = table_list.map { |table| "'#{table}'" }.join(", ")
+
+      sql = <<~SQL
+        SELECT t.relname AS table_name,
+              CASE
+                WHEN t.relreplident = 'd' THEN 'default'
+                WHEN t.relreplident = 'n' THEN 'nothing'
+                WHEN t.relreplident = 'i' THEN 'index'
+                WHEN t.relreplident = 'f' THEN 'full'
+              END AS replica_identity
+        FROM pg_class t
+        JOIN pg_namespace ns ON t.relnamespace = ns.oid
+        WHERE ns.nspname = '#{schema_name}'
+          AND t.relkind = 'r'
+          AND t.relname IN (#{formatted_table_list})
+      SQL
+
+      results =
+        Query.run(
+          query: sql,
+          connection_url: conn_string,
+          user: db_user(conn_string),
+        )
+
+      results.all? { |r| r[:replica_identity] != "nothing" }
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_easy_replicate
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-12-29 00:00:00.000000000 Z
+date: 2024-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai
@@ -47,7 +47,7 @@ dependencies:
         version: '5.69'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.76'
+        version: '5.77'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -57,7 +57,7 @@ dependencies:
         version: '5.69'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.76'
+        version: '5.77'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement