pg_easy_replicate 0.1.8 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 73d97a0f8505fef3ac73849e6964d1204c72545eb52fb4059d2c1119ea62228d
-  data.tar.gz: 8ad93a08e70da1945e091110064a9b4be569c2a77328f9960cd7ce52c6becefa
+  metadata.gz: d45fb127cf497feec36832e91ca67a54878d9fcf783cab3059aad366b0e99026
+  data.tar.gz: 989e3900bebfb04cb9af6037bf849da4f2ea9c6da9a9c6899d64a193ab507b93
 SHA512:
-  metadata.gz: 895ca04cdabbf15a082b1623890d55e650398f07eb144b2a0c8cf2877557a857926e24bfc336046afdd5097dbd5042d43d344e095338d6ffdb8f259797536c10
-  data.tar.gz: 41dd6428dd0f8c54de4d4b97931d573a0b2ae5fd4f5d61e2908d2e88e7fff0a97ed13c841a8fa2b924bacda0e8ae4c496d3f533d1398be7b15ffe0aa0ba6304a
+  metadata.gz: a5e5a55884a847b5bcfbaee5252704a60e2969c7dba7813fd35e875b2e381c65a7be5088c3042d4473acde9205b0423c59d22e0c9792178c9fd2e49e86b4ef4d
+  data.tar.gz: 17af5a3e97dc562970e3f84f1aa2eb4878159cf5bf8f4d1d7cccd3948ded00c7f2c2f003024281409e07a2d9e19bee1eab8aa7c9a5959eb7ebdc8d9c1da75f33

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## [0.1.10] - 2023-12-12
+
+- Reference the passed in URL and use source db url - #74
+
+## [0.1.9] - 2023-08-01
+
+- Exclude views, temporary tables and foreign tables from #list_all_tables - #39
+- Add quote_identifier helper for SQL identifiers. - #40
+- Escape db user name in queries - #42
+- Require english lib so that $CHILD_STATUS is loaded - #43
+- Bump rubocop from 1.54.2 to 1.55.0 - #37
+- Bump rubocop-rspec from 2.22.0 to 2.23.0 - #36
+- Quote indent username, dbname and schema in all places - #44
+
 ## [0.1.8] - 2023-07-23
 
 - Introduce --copy_schema via pg_dump - #35

data/Gemfile.lock

@@ -1,43 +1,44 @@
 PATH
   remote: .
   specs:
-    pg_easy_replicate (0.1.8)
+    pg_easy_replicate (0.1.10)
       ougai (~> 2.0.0)
       pg (~> 1.5.3)
-      sequel (>= 5.69, < 5.71)
+      sequel (>= 5.69, < 5.76)
       thor (~> 1.2.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
+    bigdecimal (3.1.4)
     coderay (1.1.3)
     diff-lcs (1.5.0)
     haml (6.1.1)
       temple (>= 0.8.2)
       thor
       tilt
-    json (2.6.3)
+    json (2.7.0)
     language_server-protocol (3.17.0.3)
     method_source (1.0.0)
     oj (3.14.3)
     ougai (2.0.0)
       oj (~> 3.10)
     parallel (1.23.0)
-    parser (3.2.2.3)
+    parser (3.2.2.4)
       ast (~> 2.4.1)
       racc
-    pg (1.5.3)
+    pg (1.5.4)
     prettier_print (1.2.1)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    racc (1.7.1)
+    racc (1.7.3)
     rainbow (3.1.1)
     rake (13.0.6)
     rbs (3.1.0)
-    regexp_parser (2.8.1)
-    rexml (3.2.5)
+    regexp_parser (2.8.2)
+    rexml (3.2.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
@@ -51,37 +52,38 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    rubocop (1.54.2)
+    rubocop (1.58.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.2.3)
+      parser (>= 3.2.2.4)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.28.0, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.29.0)
+    rubocop-ast (1.30.0)
       parser (>= 3.2.1.0)
-    rubocop-capybara (2.18.0)
+    rubocop-capybara (2.19.0)
       rubocop (~> 1.41)
-    rubocop-factory_bot (2.23.1)
+    rubocop-factory_bot (2.24.0)
       rubocop (~> 1.33)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-    rubocop-performance (1.18.0)
+    rubocop-performance (1.19.1)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
-    rubocop-rspec (2.22.0)
+    rubocop-rspec (2.24.1)
       rubocop (~> 1.33)
       rubocop-capybara (~> 2.17)
       rubocop-factory_bot (~> 2.22)
     ruby-progressbar (1.13.0)
-    sequel (5.70.0)
-    syntax_tree (6.1.1)
+    sequel (5.75.0)
+      bigdecimal
+    syntax_tree (6.2.0)
       prettier_print (>= 1.2.0)
     syntax_tree-haml (4.0.3)
       haml (>= 5.2)
@@ -94,7 +96,7 @@ GEM
     temple (0.10.1)
     thor (1.2.2)
     tilt (2.1.0)
-    unicode-display_width (2.4.2)
+    unicode-display_width (2.5.0)
 
 PLATFORMS
   arm64-darwin-22

data/README.md

@@ -4,7 +4,7 @@
 [![Smoke spec](https://github.com/shayonj/pg_easy_replicate/actions/workflows/smoke.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg_easy_replicate/actions/workflows/ci.yaml)
 [![Gem Version](https://badge.fury.io/rb/pg_easy_replicate.svg?2)](https://badge.fury.io/rb/pg_easy_replicate)
 
-`pg_easy_replicate` is a CLI orchestrator tool that simplifies the process of setting up [logical replication](https://www.postgresql.org/docs/current/logical-replication.html) between two PostgreSQL databases. `pg_easy_replicate` also supports switchover. After the source (primary database) is fully replicating, `pg_easy_replicate` puts it into read-only mode and via logical replication flushes all data to the new target database. This ensures zero data loss and minimal downtime for the application. This method can be useful for performing minimal downtime (up to <1min, depending) major version upgrades between two PostgreSQL databases, load testing with blue/green database setup and other similar use cases.
+`pg_easy_replicate` is a CLI orchestrator tool that simplifies the process of setting up [logical replication](https://www.postgresql.org/docs/current/logical-replication.html) between two PostgreSQL databases. `pg_easy_replicate` also supports switchover. After the source (primary database) is fully replicated, `pg_easy_replicate` puts it into read-only mode and via logical replication flushes all data to the new target database. This ensures zero data loss and minimal downtime for the application. This method can be useful for performing minimal downtime (up to <1min, depending) major version upgrades between two PostgreSQL databases, load testing with blue/green database setup and other similar use cases.
 
 Battle tested in production at [Tines](https://www.tines.com/) 🚀
 
@@ -58,7 +58,7 @@ https://hub.docker.com/r/shayonj/pg_easy_replicate
 
 - PostgreSQL 10 and later
 - Ruby 2.7 and later
-- Database user should have permissions for `SUPERUSER` or pass in the special user role that has the privileges to create role, schema, publication and subscription on both databases. More on `--special-user-role` section below.
+- Database users should have `SUPERUSER` permissions, or pass in a special user with privileges to create the needed role, schema, publication and subscription on both databases. More on `--special-user-role` section below.
 
 ## Limits
 
@@ -160,7 +160,7 @@ $ pg_easy_replicate start_sync --group-name database-cluster-1
 
 ### Stats
 
-You can inspect or watch stats any time during the sync process. The stats give you can an idea of when the sync started, current flush/write lag, how many tables are in `replicating`, `copying` or other stages, and more.
+You can inspect or watch stats any time during the sync process. The stats give you an idea of when the sync started, current flush/write lag, how many tables are in `replicating`, `copying` or other stages, and more.
 
 You can poll these stats to perform any other after the switchover is done. The stats include a `switchover_completed_at` which is updated once the switch over is complete.
 
@@ -203,7 +203,7 @@ $ pg_easy_replicate stats --group-name database-cluster-1
 
 `switchover` will wait until all tables in the group are replicating and the delta for lag is <200kb (by calculating the `pg_wal_lsn_diff` between `sent_lsn` and `write_lsn`) and then perform the switch.
 
-The switch is made by putting the user on the source database in `READ ONLY` mode, so that it is not accepting any more writes and waits for the flush lag to be `0`. It is up to user to kick of a rolling restart of your application containers or failover DNS (more on these below in strategies) after the switchover is complete, so that your application isn't sending any read/write requests to the old/source database.
+The switch is made by putting the user on the source database in `READ ONLY` mode, so that it is not accepting any more writes and waits for the flush lag to be `0`. It’s up to the user to kick off a rolling restart of your application containers or failover DNS (more on these below in strategies) after the switchover is complete, so that your application isn't sending any read/write requests to the old/source database.
 
 ```bash
 $ pg_easy_replicate switchover  --group-name database-cluster-1
@@ -252,7 +252,7 @@ Next, you can set up a program that watches the `stats` and waits until `switcho
 
 ### Adding internal user to pgBouncer `userlist`
 
-`pg_easy_replicate` creates a special user to orchestrate the replication. If you us pgBouncer, you may need to allow `pger_su_h1a4fb` as a user that can perform login by adding it to the `userlist`.
+`pg_easy_replicate` creates a special user to orchestrate the replication. If you use pgBouncer, you may need to allow `pger_su_h1a4fb` as a user that can perform login by adding it to the `userlist`.
 
 ## Contributing
 

data/docker-compose.yml

@@ -5,8 +5,8 @@ services:
     ports:
       - "5432:5432"
     environment:
-      POSTGRES_USER: jamesbond
-      POSTGRES_PASSWORD: jamesbond123@7!'3aaR
+      POSTGRES_USER: james-bond
+      POSTGRES_PASSWORD: james-bond123@7!'3aaR
       POSTGRES_DB: postgres
     command: >
       -c wal_level=logical
@@ -21,8 +21,8 @@ services:
     ports:
       - "5433:5432"
     environment:
-      POSTGRES_USER: jamesbond
-      POSTGRES_PASSWORD: jamesbond123@7!'3aaR
+      POSTGRES_USER: james-bond
+      POSTGRES_PASSWORD: james-bond123@7!'3aaR
       POSTGRES_DB: postgres
     command: >
       -c wal_level=logical

data/lib/pg_easy_replicate/helper.rb

@@ -43,6 +43,10 @@ module PgEasyReplicate
         .downcase
     end
 
+    def quote_ident(sql_ident)
+      PG::Connection.quote_ident(sql_ident)
+    end
+
     def test_env?
       ENV.fetch("RACK_ENV", nil) == "test"
     end

data/lib/pg_easy_replicate/orchestrate.rb

@@ -94,7 +94,8 @@ module PgEasyReplicate
           .map do |table_name|
             Query.run(
               query:
-                "ALTER PUBLICATION #{publication_name(group_name)} ADD TABLE \"#{table_name}\"",
+                "ALTER PUBLICATION #{quote_ident(publication_name(group_name))}
+                        ADD TABLE #{quote_ident(table_name)}",
               connection_url: conn_string,
               schema: schema,
             )
@@ -107,7 +108,11 @@ module PgEasyReplicate
         Query
           .run(
             query:
-              "SELECT table_name FROM information_schema.tables WHERE table_schema = '#{schema}' ORDER BY table_name",
+              "SELECT table_name
+               FROM information_schema.tables
+               WHERE table_schema = '#{schema}' AND
+                 table_type = 'BASE TABLE'
+               ORDER BY table_name",
             connection_url: conn_string,
           )
           .map(&:values)
@@ -121,7 +126,8 @@ module PgEasyReplicate
           { publication_name: publication_name(group_name) },
         )
         Query.run(
-          query: "DROP PUBLICATION IF EXISTS #{publication_name(group_name)}",
+          query:
+            "DROP PUBLICATION IF EXISTS #{quote_ident(publication_name(group_name))}",
           connection_url: conn_string,
           user: db_user(conn_string),
         )
@@ -144,7 +150,9 @@ module PgEasyReplicate
 
         Query.run(
           query:
-            "CREATE SUBSCRIPTION #{subscription_name(group_name)} CONNECTION '#{source_conn_string}' PUBLICATION #{publication_name(group_name)}",
+            "CREATE SUBSCRIPTION #{quote_ident(subscription_name(group_name))}
+                      CONNECTION '#{source_conn_string}'
+                      PUBLICATION #{quote_ident(publication_name(group_name))}",
           connection_url: target_conn_string,
           user: db_user(target_conn_string),
           transaction: false,
@@ -282,7 +290,7 @@ module PgEasyReplicate
         )
 
         alter_sql =
-          "ALTER USER #{db_user(source_db_url)} set default_transaction_read_only = true"
+          "ALTER USER #{quote_ident(db_user(source_db_url))} set default_transaction_read_only = true"
         Query.run(query: alter_sql, connection_url: source_db_url)
 
         kill_sql =
@@ -297,7 +305,7 @@ module PgEasyReplicate
         logger.info("Restoring connections")
 
         alter_sql =
-          "ALTER USER #{db_user(source_db_url)} set default_transaction_read_only = false"
+          "ALTER USER #{quote_ident(db_user(source_db_url))} set default_transaction_read_only = false"
         Query.run(query: alter_sql, connection_url: source_db_url)
       end
 

data/lib/pg_easy_replicate/query.rb

@@ -17,12 +17,12 @@ module PgEasyReplicate
         if transaction
           r =
             conn.transaction do
-              conn.run("SET search_path to #{schema}") if schema
+              conn.run("SET search_path to #{quote_ident(schema)}") if schema
               conn.run("SET statement_timeout to '5s'")
               conn.fetch(query).to_a
             end
         else
-          conn.run("SET search_path to #{schema}") if schema
+          conn.run("SET search_path to #{quote_ident(schema)}") if schema
           conn.run("SET statement_timeout to '5s'")
           r = conn.fetch(query).to_a
         end

data/lib/pg_easy_replicate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEasyReplicate
-  VERSION = "0.1.8"
+  VERSION = "0.1.10"
 end

data/lib/pg_easy_replicate.rb

@@ -5,6 +5,7 @@ require "ougai"
 require "pg"
 require "sequel"
 require "open3"
+require "English"
 
 require "pg_easy_replicate/helper"
 require "pg_easy_replicate/version"
@@ -99,7 +100,6 @@ module PgEasyReplicate
       logger.info("Setting up replication user on source database")
       create_user(
         conn_string: source_db_url,
-        group_name: options[:group_name],
         special_user_role: options[:special_user_role],
         grant_permissions_on_schema: true,
       )
@@ -107,7 +107,6 @@ module PgEasyReplicate
       logger.info("Setting up replication user on target database")
       create_user(
         conn_string: target_db_url,
-        group_name: options[:group_name],
         special_user_role: options[:special_user_role],
       )
 
@@ -152,10 +151,11 @@ module PgEasyReplicate
 
     def drop_internal_schema
       Query.run(
-        query: "DROP SCHEMA IF EXISTS #{internal_schema_name} CASCADE",
+        query:
+          "DROP SCHEMA IF EXISTS #{quote_ident(internal_schema_name)} CASCADE",
         connection_url: source_db_url,
         schema: internal_schema_name,
-        user: db_user(target_db_url),
+        user: db_user(source_db_url),
       )
     rescue => e
       raise "Unable to drop schema: #{e.message}"
@@ -163,16 +163,16 @@ module PgEasyReplicate
 
     def setup_internal_schema
       sql = <<~SQL
-        create schema if not exists #{internal_schema_name};
-        grant usage on schema #{internal_schema_name} to #{db_user(source_db_url)};
-        grant create on schema #{internal_schema_name} to #{db_user(source_db_url)};
+        create schema if not exists #{quote_ident(internal_schema_name)};
+        grant usage on schema #{quote_ident(internal_schema_name)} to #{quote_ident(db_user(source_db_url))};
+        grant create on schema #{quote_ident(internal_schema_name)} to #{quote_ident(db_user(source_db_url))};
       SQL
 
       Query.run(
         query: sql,
         connection_url: source_db_url,
         schema: internal_schema_name,
-        user: db_user(target_db_url),
+        user: db_user(source_db_url),
       )
     rescue => e
       raise "Unable to setup schema: #{e.message}"
@@ -241,12 +241,7 @@ module PgEasyReplicate
           ORDER BY 1;
         SQL
 
-        r =
-          Query.run(
-            query: sql,
-            connection_url: url,
-            user: db_user(target_db_url),
-          )
+        r = Query.run(query: sql, connection_url: url, user: db_user(url))
         # If special_user_role is passed just ensure the url in conn_string has been granted
         # the special_user_role
         r.any? { |q| q[:role] == special_user_role }
@@ -256,7 +251,7 @@ module PgEasyReplicate
             query:
               "SELECT rolname, rolsuper FROM pg_roles where rolname = '#{db_user(url)}';",
             connection_url: url,
-            user: db_user(target_db_url),
+            user: db_user(url),
           )
         r.any? { |q| q[:rolsuper] }
       end
@@ -266,16 +261,15 @@ module PgEasyReplicate
 
     def create_user(
       conn_string:,
-      group_name:,
       special_user_role: nil,
       grant_permissions_on_schema: false
     )
       password = connection_info(conn_string)[:password].gsub("'") { "''" }
 
       sql = <<~SQL
-        drop role if exists #{internal_user_name};
-        create role #{internal_user_name} with password '#{password}' login createdb createrole;
-        grant all privileges on database #{db_name(conn_string)} TO #{internal_user_name};
+        drop role if exists #{quote_ident(internal_user_name)};
+        create role #{quote_ident(internal_user_name)} with password '#{password}' login createdb createrole;
+        grant all privileges on database #{quote_ident(db_name(conn_string))} TO #{quote_ident(internal_user_name)};
       SQL
 
       Query.run(
@@ -287,9 +281,9 @@ module PgEasyReplicate
 
       sql =
         if special_user_role
-          "grant #{special_user_role} to #{internal_user_name};"
+          "grant #{quote_ident(special_user_role)} to #{quote_ident(internal_user_name)};"
         else
-          "alter user #{internal_user_name} with superuser;"
+          "alter user #{quote_ident(internal_user_name)} with superuser;"
         end
 
       Query.run(
@@ -302,7 +296,7 @@ module PgEasyReplicate
       return unless grant_permissions_on_schema
       Query.run(
         query:
-          "grant all on schema #{internal_schema_name} to #{internal_user_name}",
+          "grant all on schema #{quote_ident(internal_schema_name)} to #{quote_ident(internal_user_name)}",
         connection_url: conn_string,
         user: db_user(conn_string),
         transaction: false,
@@ -313,7 +307,7 @@ module PgEasyReplicate
 
     def drop_user(conn_string:, group_name:)
       sql = <<~SQL
-       revoke all privileges on database #{db_name(conn_string)} from #{internal_user_name};
+       revoke all privileges on database #{db_name(conn_string)} from #{quote_ident(internal_user_name)};
       SQL
       Query.run(
         query: sql,
@@ -322,7 +316,7 @@ module PgEasyReplicate
       )
 
       sql = <<~SQL
-        drop role if exists #{internal_user_name};
+        drop role if exists #{quote_ident(internal_user_name)};
       SQL
 
       Query.run(

data/scripts/e2e-bootstrap.sh

@@ -3,13 +3,13 @@
 set -eo pipefail
 
 if [[ -z ${GITHUB_WORKFLOW} ]]; then
-  export SECONDARY_SOURCE_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@source_db/postgres"
+  export SECONDARY_SOURCE_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@source_db/postgres"
 fi
 
-export SOURCE_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@localhost:5432/postgres"
-export TARGET_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@localhost:5433/postgres"
-export PGPASSWORD='jamesbond123@7!'"'"'3aaR'
+export SOURCE_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhost:5432/postgres"
+export TARGET_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhost:5433/postgres"
+export PGPASSWORD='james-bond123@7!'"'"'3aaR'
 
-pgbench --initialize -s 5 --foreign-keys --host localhost -U jamesbond -d postgres
+pgbench --initialize -s 5 --foreign-keys --host localhost -U james-bond -d postgres
 
 bundle exec bin/pg_easy_replicate config_check --copy-schema

data/scripts/e2e-start.sh

@@ -3,12 +3,12 @@
 set -eo pipefail
 
 if [[ -z ${GITHUB_WORKFLOW} ]]; then
-  export SECONDARY_SOURCE_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@source_db/postgres"
+  export SECONDARY_SOURCE_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@source_db/postgres"
 fi
 
-export SOURCE_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@localhost:5432/postgres"
-export TARGET_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@localhost:5433/postgres"
-export PGPASSWORD='jamesbond123@7!'"'"''"'"'3aaR'
+export SOURCE_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhost:5432/postgres"
+export TARGET_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhost:5433/postgres"
+export PGPASSWORD='james-bond123@7!'"'"''"'"'3aaR'
 
 # Bootstrap and cleanup
 echo "===== Performing Bootstrap and cleanup"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_easy_replicate
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.10
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-23 00:00:00.000000000 Z
+date: 2023-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai
@@ -47,7 +47,7 @@ dependencies:
         version: '5.69'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.71'
+        version: '5.76'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -57,7 +57,7 @@ dependencies:
         version: '5.69'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.71'
+        version: '5.76'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement