pg_easy_replicate 0.2.5 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e6514e70589083375b9f9112d737a2620f48d62d07d4cc5ec213c209289a1ad
-  data.tar.gz: fe93ff6a1a741c8f330d878893e511aa0e82e5505acaa031e5b809b482553999
+  metadata.gz: 90a3489ff7f33639118b99b142c2dcdef491ad384df80de0cceee89b706a9073
+  data.tar.gz: 0ab288970a9eb6465878ef03aca0d67f7543426a4cc744a4e47592ac200f5b68
 SHA512:
-  metadata.gz: c14eb4c2256e0adb336a628d24d88a4134849f75f2ca3a656ef3abc9474de3a4d5b06dae994f826674e3343998c34d69604e0a8d417cfee2b7e9675d9b77b0c6
-  data.tar.gz: 590c021623ea48f17059cb5b7f640a82433f4f25c99ae1afdc20544d751271f7a57d6ff8243328aa611ca9acb61969660b4eb5b82cd976147727c934ce76f8e3
+  metadata.gz: e487ac3b147855379c230d3b792d702d42776eb254f227cf31a8ebc5dee677cbf47b2efd7a9c0c218a87f064b1449e3c286d1dbc6c0f50f6b108cee3b1a5ccaa
+  data.tar.gz: 5ce88fff42f5ba68977b15261376d48e23173d210eddd1beccdad148cafab45f8e760a09d37991cef491313b4dee12dfa761b495fa1b90e8348c2b89ed2c3a6a

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## [0.2.6] - 2024-06-04
+
+- Quote table name in the VACUUM SQL - #118
+- Exclude tables created by extensions - #120
+- Use db user when adding tables to replication - #130
+
+## [0.2.5] - 2024-04-14
+
+- List only permanent tables - #113
+
 ## [0.2.4] - 2024-02-13
 
 - Introduce PG_EASY_REPLICATE_STATEMENT_TIMEOUT env var

data/Gemfile.lock

@@ -1,17 +1,17 @@
 PATH
   remote: .
   specs:
-    pg_easy_replicate (0.2.5)
+    pg_easy_replicate (0.2.7)
       ougai (~> 2.0.0)
       pg (~> 1.5.3)
-      sequel (>= 5.69, < 5.80)
+      sequel (>= 5.69, < 5.83)
       thor (>= 1.2.2, < 1.4.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
-    bigdecimal (3.1.7)
+    bigdecimal (3.1.8)
     coderay (1.1.3)
     diff-lcs (1.5.1)
     haml (6.1.1)
@@ -24,21 +24,22 @@ GEM
     oj (3.14.3)
     ougai (2.0.0)
       oj (~> 3.10)
-    parallel (1.24.0)
-    parser (3.3.0.5)
+    parallel (1.25.1)
+    parser (3.3.3.0)
       ast (~> 2.4.1)
       racc
-    pg (1.5.6)
+    pg (1.5.7)
     prettier_print (1.2.1)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    racc (1.7.3)
+    racc (1.8.0)
     rainbow (3.1.1)
-    rake (13.1.0)
+    rake (13.2.1)
     rbs (3.1.0)
-    regexp_parser (2.9.0)
-    rexml (3.2.6)
+    regexp_parser (2.9.2)
+    rexml (3.3.3)
+      strscan
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
@@ -52,7 +53,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.0)
-    rubocop (1.62.1)
+    rubocop (1.64.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -63,17 +64,17 @@ GEM
       rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.2)
-      parser (>= 3.3.0.4)
+    rubocop-ast (1.31.3)
+      parser (>= 3.3.1.0)
     rubocop-capybara (2.20.0)
       rubocop (~> 1.41)
     rubocop-factory_bot (2.25.1)
       rubocop (~> 1.41)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-    rubocop-performance (1.20.2)
+    rubocop-performance (1.21.1)
       rubocop (>= 1.48.1, < 2.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
     rubocop-rspec (2.29.1)
@@ -84,8 +85,9 @@ GEM
     rubocop-rspec_rails (2.28.2)
       rubocop (~> 1.40)
     ruby-progressbar (1.13.0)
-    sequel (5.79.0)
+    sequel (5.82.0)
       bigdecimal
+    strscan (3.1.0)
     syntax_tree (6.2.0)
       prettier_print (>= 1.2.0)
     syntax_tree-haml (4.0.3)

data/README.md

@@ -69,8 +69,11 @@ All [Logical Replication Restrictions](https://www.postgresql.org/docs/current/l
 
 ## Usage
 
-Ensure `SOURCE_DB_URL` and `TARGET_DB_URL` are present as environment variables in the runtime environment. The URL are of the postgres connection string format. Example:
+Ensure `SOURCE_DB_URL` and `TARGET_DB_URL` are present as environment variables in the runtime environment. 
+- `SOURCE_DB_URL` = The database that you want to replicate FROM.
+- `TARGET_DB_URL` = The database that you want to replicate TO.
 
+The URL should be in postgres connection string format. Example:
 ```bash
 $ export SOURCE_DB_URL="postgres://USERNAME:PASSWORD@localhost:5432/DATABASE_NAME"
 $ export TARGET_DB_URL="postgres://USERNAME:PASSWORD@localhost:5433/DATABASE_NAME"
@@ -84,6 +87,12 @@ You can extend the default timeout by setting the following environment variable
 $ export PG_EASY_REPLICATE_STATEMENT_TIMEOUT="10s" # default 5s
 ```
 
+You can get additional debug logging by adding the following environment variable
+
+```bash
+$ export DEBUG="true"
+```
+
 Any `pg_easy_replicate` command can be run the same way with the docker image as well. As long the container is running in an environment where it has access to both the databases. Example
 
 ```bash
@@ -247,6 +256,30 @@ $ pg_easy_replicate switchover  --group-name database-cluster-2
 ...
 ```
 
+## Exclude tables from replication
+
+By default all tables are added for replication but you can exclude tables if necessary. Example
+
+```bash
+...
+$ pg_easy_replicate bootstrap --group-name database-cluster-1 --copy-schema
+$ pg_easy_replicate start_sync --group-name database-cluster-1 --schema-name public --exclude_tables "events"
+...
+```
+
+### Cleanup
+
+Use `cleanup` if you want to remove all bootstrapped data for the specified group. Additionally you can pass `-e` or `--everything` in order to clean up all schema changes for bootstrapped tables, users and any publication/subscription data.
+
+```bash
+$ pg_easy_replicate cleanup  --group-name database-cluster-1 --everything
+
+{"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":24192,"level":30,"time":"2023-06-19T16:05:23.033-04:00","v":0,"msg":"Dropping groups table","version":"0.1.0"}
+
+{"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":24192,"level":30,"time":"2023-06-19T16:05:23.033-04:00","v":0,"msg":"Dropping schema","version":"0.1.0"}
+...
+```
+
 ## Switchover strategies with minimal downtime
 
 For minimal downtime, it'd be best to watch/tail the stats and wait until `switchover_completed_at` is updated with a timestamp. Once that happens you can perform any of the following strategies. Note: These are just suggestions and `pg_easy_replicate` doesn't provide any functionalities for this.

data/lib/pg_easy_replicate/cli.rb

@@ -21,6 +21,11 @@ module PgEasyReplicate
                   default: "",
                   desc:
                     "Comma separated list of table names. Default: All tables"
+    method_option :exclude_tables,
+                  aliases: "-e",
+                  default: "",
+                  desc:
+                    "Comma separated list of table names to exclude. Default: None"
     method_option :schema_name,
                   aliases: "-s",
                   desc:
@@ -30,6 +35,7 @@ module PgEasyReplicate
         special_user_role: options[:special_user_role],
         copy_schema: options[:copy_schema],
         tables: options[:tables],
+        exclude_tables: options[:exclude_tables],
         schema_name: options[:schema_name],
       )
 
@@ -91,6 +97,11 @@ module PgEasyReplicate
                   default: "",
                   desc:
                     "Comma separated list of table names. Default: All tables"
+    method_option :exclude_tables,
+                  aliases: "-e",
+                  default: "",
+                  desc:
+                    "Comma separated list of table names to exclude. Default: None"
     method_option :recreate_indices_post_copy,
                   type: :boolean,
                   default: true,

data/lib/pg_easy_replicate/helper.rb

@@ -73,14 +73,18 @@ module PgEasyReplicate
       abort(msg)
     end
 
-    def determine_tables(conn_string:, list: "", schema: nil)
+    def determine_tables(conn_string:, list: "", exclude_list: "", schema: nil)
       schema ||= "public"
-
-      tables = list&.split(",") || []
-      if tables.size > 0
-        tables
+      
+      tables = convert_to_array(list)
+      exclude_tables = convert_to_array(exclude_list)
+      validate_table_lists(tables, exclude_tables, schema)
+
+      if tables.empty?
+        all_tables = list_all_tables(schema: schema, conn_string: conn_string)
+        all_tables - (exclude_tables + %w[spatial_ref_sys])
       else
-        list_all_tables(schema: schema, conn_string: conn_string)
+        tables
       end
     end
 
@@ -101,5 +105,24 @@ module PgEasyReplicate
         .map(&:values)
         .flatten
     end
+
+    def convert_to_array(input)
+      input.is_a?(Array) ? input : input&.split(",") || []
+    end
+
+    def validate_table_lists(tables, exclude_tables, schema_name)
+      table_list = convert_to_array(tables)
+      exclude_table_list = convert_to_array(exclude_tables)
+
+      if !table_list.empty? && !exclude_table_list.empty?
+        abort_with("Options --tables(-t) and --exclude-tables(-e) cannot be used together.")
+      elsif !table_list.empty?
+        if table_list.size > 0 && (schema_name.nil? || schema_name == "")
+          abort_with("Schema name is required if tables are passed")
+        end
+      elsif exclude_table_list.size > 0 && (schema_name.nil? || schema_name == "")
+        abort_with("Schema name is required if exclude tables are passed")
+      end
+    end
   end
 end

data/lib/pg_easy_replicate/orchestrate.rb

@@ -15,6 +15,7 @@ module PgEasyReplicate
             schema: schema_name,
             conn_string: source_db_url,
             list: options[:tables],
+            exclude_list: options[:exclude_tables]
           )
 
         if options[:recreate_indices_post_copy]
@@ -106,6 +107,7 @@ module PgEasyReplicate
                         ADD TABLE #{quote_ident(table_name)}",
             connection_url: conn_string,
             schema: schema,
+            user: db_user(conn_string),
           )
         end
       rescue => e
@@ -365,7 +367,7 @@ module PgEasyReplicate
           )
 
           Query.run(
-            query: "VACUUM VERBOSE ANALYZE #{t};",
+            query: "VACUUM VERBOSE ANALYZE #{quote_ident(t)};",
             connection_url: conn_string,
             schema: schema,
             transaction: false,

data/lib/pg_easy_replicate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEasyReplicate
-  VERSION = "0.2.5"
+  VERSION = "0.2.7"
 end

data/lib/pg_easy_replicate.rb

@@ -30,11 +30,18 @@ module PgEasyReplicate
       special_user_role: nil,
       copy_schema: false,
       tables: "",
+      exclude_tables: "",
       schema_name: nil
     )
       abort_with("SOURCE_DB_URL is missing") if source_db_url.nil?
       abort_with("TARGET_DB_URL is missing") if target_db_url.nil?
 
+      if !tables.empty? && !exclude_tables.empty?
+        abort_with(
+          "Options --tables(-t) and --exclude-tables(-e) cannot be used together.",
+        )
+      end
+
       system("which pg_dump")
       pg_dump_exists = $CHILD_STATUS.success?
 
@@ -65,6 +72,7 @@ module PgEasyReplicate
               tables_have_replica_identity?(
                 conn_string: source_db_url,
                 tables: tables,
+                exclude_tables: exclude_tables,
                 schema_name: schema_name,
               ),
           }
@@ -77,6 +85,7 @@ module PgEasyReplicate
       special_user_role: nil,
       copy_schema: false,
       tables: "",
+      exclude_tables: "",
       schema_name: nil
     )
       config_hash =
@@ -84,6 +93,7 @@ module PgEasyReplicate
           special_user_role: special_user_role,
           copy_schema: copy_schema,
           tables: tables,
+          exclude_tables: exclude_tables,
           schema_name: schema_name,
         )
 
@@ -103,9 +113,7 @@ module PgEasyReplicate
         abort_with("User on source database does not have super user privilege")
       end
 
-      if tables.split(",").size > 0 && (schema_name.nil? || schema_name == "")
-        abort_with("Schema name is required if tables are passed")
-      end
+      validate_table_lists(tables, exclude_tables, schema_name)
 
       unless config_hash.dig(:tables_have_replica_identity)
         abort_with(
@@ -152,34 +160,58 @@ module PgEasyReplicate
     end
 
     def cleanup(options)
-      logger.info("Dropping groups table")
-      Group.drop
-
-      if options[:everything]
-        logger.info("Dropping schema")
-        drop_internal_schema
-      end
-
-      if options[:everything] || options[:sync]
-        Orchestrate.drop_publication(
-          group_name: options[:group_name],
-          conn_string: source_db_url,
-        )
-
-        Orchestrate.drop_subscription(
-          group_name: options[:group_name],
-          target_conn_string: target_db_url,
+      cleanup_steps = [
+        -> do
+          logger.info("Dropping groups table")
+          Group.drop
+        end,
+        -> do
+          if options[:everything]
+            logger.info("Dropping schema")
+            drop_internal_schema
+          end
+        end,
+        -> do
+          if options[:everything] || options[:sync]
+            logger.info("Dropping publication on source database")
+            Orchestrate.drop_publication(
+              group_name: options[:group_name],
+              conn_string: source_db_url,
+            )
+          end
+        end,
+        -> do
+          if options[:everything] || options[:sync]
+            logger.info("Dropping subscription on target database")
+            Orchestrate.drop_subscription(
+              group_name: options[:group_name],
+              target_conn_string: target_db_url,
+            )
+          end
+        end,
+        -> do
+          if options[:everything]
+            logger.info("Dropping replication user on source database")
+            drop_user(conn_string: source_db_url)
+          end
+        end,
+        -> do
+          if options[:everything]
+            logger.info("Dropping replication user on target database")
+            drop_user(conn_string: target_db_url)
+          end
+        end,
+      ]
+
+      cleanup_steps.each do |step|
+        step.call
+      rescue => e
+        logger.warn(
+          "Part of the cleanup step failed with #{e.message}. Continuing...",
         )
       end
 
-      if options[:everything]
-        # Drop users at last
-        logger.info("Dropping replication user on source database")
-        drop_user(conn_string: source_db_url)
-
-        logger.info("Dropping replication user on target database")
-        drop_user(conn_string: target_db_url)
-      end
+      logger.info("Cleanup process completed.")
     rescue => e
       abort_with("Unable to cleanup: #{e.message}")
     end
@@ -299,9 +331,9 @@ module PgEasyReplicate
       special_user_role: nil,
       grant_permissions_on_schema: false
     )
-      password = connection_info(conn_string)[:password].gsub("'") { "''" }
+      return if user_exists?(conn_string: conn_string, user: internal_user_name)
 
-      drop_user(conn_string: conn_string)
+      password = connection_info(conn_string)[:password].gsub("'") { "''" }
 
       sql = <<~SQL
         create role #{quote_ident(internal_user_name)} with password '#{password}' login createdb createrole;
@@ -390,6 +422,7 @@ module PgEasyReplicate
     def tables_have_replica_identity?(
       conn_string:,
       tables: "",
+      exclude_tables: "",
       schema_name: nil
     )
       schema_name ||= "public"
@@ -399,6 +432,7 @@ module PgEasyReplicate
           schema: schema_name,
           conn_string: source_db_url,
           list: tables,
+          exclude_list: exclude_tables,
         )
       return false if table_list.empty?
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_easy_replicate
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.2.7
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-04-14 00:00:00.000000000 Z
+date: 2024-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai
@@ -47,7 +47,7 @@ dependencies:
         version: '5.69'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.80'
+        version: '5.83'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -57,7 +57,7 @@ dependencies:
         version: '5.69'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.80'
+        version: '5.83'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement