pg_easy_replicate 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3e58d75f6e22242f3b833838eba69bfd72cb930ad5931ef77a5c0c4f49aded8
-  data.tar.gz: 8ff3c66b8ee6b37bcadb801f57fa668019fc7c36c5cabf5639c65d3b80813eae
+  metadata.gz: f655d3bf7d458de6b84a85930e7175c05f780a390e9431531a0ae30c35d4737b
+  data.tar.gz: fa23c33715823598d4795ba64cbf466088d01fd97315bfe97d3a05db2da50f48
 SHA512:
-  metadata.gz: d337088ee7ccf6855c45328502c587172cd96fb4b57bb386a6774665277d5753533f7876fadc28d6088ab0ecfb4cc9e201245d8a05f83ca178f9dd6376df4a00
-  data.tar.gz: 8d44166f5ad25773462e952109b315195229e1127639296e3f4f56341823af60fd7feba3bef1ffcbd32cbb864cab28117bbedda3ceb9949aa96451296bc63a78
+  metadata.gz: 1e69d89ca00ca077784aab0c042e46c85faf7e15f2e585a65321084c5b3543502a7c9a440faa17f829848015a31ed50393129a0d7bfa083a27a731af4c179b68
+  data.tar.gz: ce29729ddcf74420818c0b4d86b98de28fbefd0690368ee4fcdd40f689dcf1ba6c3135850ab1e22de2bbd977c733ca5449cf42ccef728fda20c54b102b38f965

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## [0.1.4] - 2023-06-22
+
+- Drop lockbox dependency
+- Support password with special chars and test for url encoded URI
+- Support AWS and GCP special user scenarios and introduce `--special-user-role`
+
+## [0.1.3] - 2023-06-22
+
+- Docker multi-platform image build support for linux/amd64 and linux/arm64 starting 0.1.3
+
 ## [0.1.2] - 2023-06-22
 
 - Keep the internal username unique

data/Gemfile.lock

@@ -1,8 +1,7 @@
 PATH
   remote: .
   specs:
-    pg_easy_replicate (0.1.3)
-      lockbox (~> 1.2.0)
+    pg_easy_replicate (0.1.5)
       ougai (~> 2.0.0)
       pg (~> 1.5.3)
       sequel (~> 5.69.0)
@@ -19,7 +18,6 @@ GEM
       thor
       tilt
     json (2.6.3)
-    lockbox (1.2.0)
     method_source (1.0.0)
     oj (3.14.3)
     ougai (2.0.0)

data/README.md

@@ -15,6 +15,9 @@ Battle tested in production at [Tines](https://www.tines.com/) 🚀
 - [Replicating all tables with a single group](#replicating-all-tables-with-a-single-group)
   - [Config check](#config-check)
   - [Bootstrap](#bootstrap)
+  - [Bootstrap and Config Check with special user role in AWS or GCP](#bootstrap-and-config-check-with-special-user-role-in-aws-or-gcp)
+    - [Config Check](#config-check)
+    - [Bootstrap](#bootstrap-1)
   - [Start sync](#start-sync)
   - [Stats](#stats)
   - [Performing switchover](#performing-switchover)
@@ -22,6 +25,7 @@ Battle tested in production at [Tines](https://www.tines.com/) 🚀
 - [Switchover strategies with minimal downtime](#switchover-strategies-with-minimal-downtime)
   - [Rolling restart strategy](#rolling-restart-strategy)
   - [DNS Failover strategy](#dns-failover-strategy)
+- [Contributing](#contributing)
 
 ## Installation
 
@@ -51,7 +55,7 @@ https://hub.docker.com/r/shayonj/pg_easy_replicate
 
 - PostgreSQL 10 and later
 - Ruby 2.7 and later
-- Database user should have permissions for `SUPERUSER`
+- Database user should have permissions for `SUPERUSER` or pass in the special user role that has the privileges to create role, schema, publication and subscription on both databases. More on `--special-user-role` section below.
 - Both databases should have the same schema
 
 ## Limits
@@ -116,6 +120,31 @@ $ pg_easy_replicate bootstrap --group-name database-cluster-1
 ...
 ```
 
+### Bootstrap and Config Check with special user role in AWS or GCP
+
+If you don't want your primary login user to have `superuser` privileges or you are on AWS or GCP, you will need to pass in the special user role that has the privileges to create role, schema, publication and subscription. This is required so `pg_easy_replicate` can create a dedicated user for replication which is granted the respective special user role to carry out its functionalities.
+
+For AWS the special user role is `rds_superuser`, and for GCP it is `cloudsqlsuperuser`. Please refer to docs for the most up to date information.
+
+**Note**: The user in the connection url must be part of the special user role being supplied.
+
+#### Config Check
+
+```bash
+$ pg_easy_replicate config_check --special-user-role="rds_superuser"
+
+✅ Config is looking good.
+```
+
+#### Bootstrap
+
+```bash
+$ pg_easy_replicate bootstrap --group-name database-cluster-1 --special-user-role="rds_superuser"
+
+{"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":21485,"level":30,"time":"2023-06-19T15:51:11.015-04:00","v":0,"msg":"Setting up schema","version":"0.1.0"}
+...
+```
+
 ### Start sync
 
 Once the bootstrap is complete, you can start the sync. Starting the sync sets up the publication, subscription and performs other minor housekeeping things.

data/bin/release.sh

@@ -1,3 +1,7 @@
+#!/bin/bash
+
+set -euo pipefail
+
 export VERSION=$1
 echo "VERSION: ${VERSION}"
 
@@ -7,12 +11,12 @@ gem build pg_easy_replicate.gemspec
 echo "=== Pushing gem ===="
 gem push pg_easy_replicate-"$VERSION".gem
 
-echo "=== Sleeping for 5s ===="
-sleep 5
+echo "=== Sleeping for 15s ===="
+sleep 15
 
-# echo "=== Pushing tags to github ===="
-# git tag v"$VERSION"
-# git push origin --tags
+echo "=== Pushing tags to github ===="
+git tag v"$VERSION"
+git push origin --tags
 
-# echo "=== Cleaning up ===="
-# rm pg_easy_replicate-"$VERSION".gem
+echo "=== Cleaning up ===="
+rm pg_easy_replicate-"$VERSION".gem

data/bin/test.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -euo pipefail
+
+export SECONDARY_SOURCE_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@source_db/postgres"
+export SOURCE_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@localhost:5432/postgres"
+export TARGET_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@localhost:5433/postgres"
+
+bundle exec bin/pg_easy_replicate config_check
+
+# Bootstrap and cleanup
+echo "===== Performing Bootstrap and cleanup"
+bundle exec bin/pg_easy_replicate bootstrap -g cluster-1
+bundle exec bin/pg_easy_replicate cleanup -e -g cluster-1
+
+# Bootstrap and start_sync
+echo "===== Performing Bootstrap, start_sync, stop_sync and cleanup"
+bundle exec bin/pg_easy_replicate bootstrap -g cluster-1
+bundle exec bin/pg_easy_replicate start_sync -g cluster-1
+bundle exec bin/pg_easy_replicate stop_sync -g cluster-1
+bundle exec bin/pg_easy_replicate cleanup -e -g cluster-1
+
+# Bootstrap with switchover
+echo "===== Performing Bootstrap, start_sync, stop_sync and cleanup"
+bundle exec bin/pg_easy_replicate bootstrap -g cluster-1
+bundle exec bin/pg_easy_replicate start_sync -g cluster-1
+# bundle exec bin/pg_easy_replicate switchover -g cluster-1
+bundle exec bin/pg_easy_replicate cleanup -e -g cluster-1

data/docker-compose.yml

@@ -6,7 +6,7 @@ services:
       - "5432:5432"
     environment:
       POSTGRES_USER: jamesbond
-      POSTGRES_PASSWORD: jamesbond
+      POSTGRES_PASSWORD: jamesbond123@7!'3aaR
       POSTGRES_DB: postgres
     command: >
       -c wal_level=logical
@@ -22,7 +22,7 @@ services:
       - "5433:5432"
     environment:
       POSTGRES_USER: jamesbond
-      POSTGRES_PASSWORD: jamesbond
+      POSTGRES_PASSWORD: jamesbond123@7!'3aaR
       POSTGRES_DB: postgres
     command: >
       -c wal_level=logical

data/lib/pg_easy_replicate/cli.rb

@@ -8,8 +8,14 @@ module PgEasyReplicate
 
     desc "config_check",
          "Prints if source and target database have the required config"
+    method_option :special_user_role,
+                  aliases: "-s",
+                  desc:
+                    "Name of the role that has superuser permissions. Usually useful for AWS (rds_superuser) or GCP (cloudsqlsuperuser)."
     def config_check
-      PgEasyReplicate.assert_config
+      PgEasyReplicate.assert_config(
+        special_user_role: options[:special_user_role],
+      )
 
       puts "✅ Config is looking good."
     end
@@ -18,6 +24,10 @@ module PgEasyReplicate
                   aliases: "-g",
                   required: true,
                   desc: "Name of the group to provision"
+    method_option :special_user_role,
+                  aliases: "-s",
+                  desc:
+                    "Name of the role that has superuser permissions. Usually useful with AWS (rds_superuser) or GCP (cloudsqlsuperuser)."
     desc "bootstrap",
          "Sets up temporary tables for information required during runtime"
     def bootstrap
@@ -71,7 +81,7 @@ module PgEasyReplicate
                   required: true,
                   desc: "Name of the group previously provisioned"
     def stop_sync
-      PgEasyReplicate::Orchestrate.stop_sync(options[:group_name])
+      PgEasyReplicate::Orchestrate.stop_sync(group_name: options[:group_name])
     end
 
     desc "switchover ",

data/lib/pg_easy_replicate/helper.rb

@@ -60,6 +60,10 @@ module PgEasyReplicate
       connection_info(url)[:user]
     end
 
+    def db_name(url)
+      connection_info(url)[:dbname]
+    end
+
     def abort_with(msg)
       raise(msg) if test_env?
       abort(msg)

data/lib/pg_easy_replicate/orchestrate.rb

@@ -9,8 +9,6 @@ module PgEasyReplicate
       DEFAULT_WAIT = 5 # seconds
 
       def start_sync(options)
-        PgEasyReplicate.assert_config
-
         create_publication(
           group_name: options[:group_name],
           conn_string: source_db_url,
@@ -62,10 +60,14 @@ module PgEasyReplicate
           "Setting up publication",
           { publication_name: publication_name(group_name) },
         )
+
         Query.run(
           query: "create publication #{publication_name(group_name)}",
           connection_url: conn_string,
+          user: db_user(conn_string),
         )
+      rescue => e
+        raise "Unable to create publication: #{e.message}"
       end
 
       def add_tables_to_publication(
@@ -91,6 +93,8 @@ module PgEasyReplicate
             schema: schema,
           )
         end
+      rescue => e
+        raise "Unable to add tables to publication: #{e.message}"
       end
 
       def list_all_tables(schema:, conn_string:)
@@ -112,7 +116,10 @@ module PgEasyReplicate
         Query.run(
           query: "DROP PUBLICATION IF EXISTS #{publication_name(group_name)}",
           connection_url: conn_string,
+          user: db_user(conn_string),
         )
+      rescue => e
+        raise "Unable to drop publication: #{e.message}"
       end
 
       def create_subscription(
@@ -132,6 +139,7 @@ module PgEasyReplicate
           query:
             "CREATE SUBSCRIPTION #{subscription_name(group_name)} CONNECTION '#{source_conn_string}' PUBLICATION #{publication_name(group_name)}",
           connection_url: target_conn_string,
+          user: db_user(target_conn_string),
           transaction: false,
         )
       rescue Sequel::DatabaseError => e
@@ -141,7 +149,7 @@ module PgEasyReplicate
           )
         end
 
-        raise
+        raise "Unable to create subscription: #{e.message}"
       end
 
       def drop_subscription(group_name:, target_conn_string:)
@@ -157,11 +165,15 @@ module PgEasyReplicate
           connection_url: target_conn_string,
           transaction: false,
         )
+      rescue => e
+        raise "Unable to drop subscription: #{e.message}"
       end
 
-      def stop_sync(target_conn_string:, source_conn_string:, group_name:)
-        PgEasyReplicate.assert_config
-
+      def stop_sync(
+        group_name:,
+        source_conn_string: source_db_url,
+        target_conn_string: target_db_url
+      )
         logger.info(
           "Stopping sync",
           {
@@ -177,6 +189,8 @@ module PgEasyReplicate
           group_name: group_name,
           target_conn_string: target_conn_string,
         )
+      rescue => e
+        raise "Unable to stop sync user: #{e.message}"
       end
 
       def switchover(
@@ -185,7 +199,6 @@ module PgEasyReplicate
         target_conn_string: target_db_url,
         lag_delta_size: DEFAULT_LAG
       )
-        PgEasyReplicate.assert_config
         group = Group.find(group_name)
 
         watch_lag(group_name: group_name, lag: lag_delta_size)
@@ -258,6 +271,8 @@ module PgEasyReplicate
           "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE usename = '#{db_user(source_db_url)}';"
 
         Query.run(query: kill_sql, connection_url: source_db_url)
+      rescue => e
+        raise "Unable to revoke connections on source db: #{e.message}"
       end
 
       def restore_connections_on_source_db(group_name)
@@ -298,6 +313,8 @@ module PgEasyReplicate
         SQL
 
         Query.run(query: sql, connection_url: conn_string, schema: schema)
+      rescue => e
+        raise "Unable to refresh sequences: #{e.message}"
       end
 
       def mark_switchover_complete(group_name)

data/lib/pg_easy_replicate/stats.rb

@@ -13,7 +13,6 @@ module PgEasyReplicate
 
     class << self
       def object(group_name)
-        PgEasyReplicate.assert_config
         stats = replication_stats(group_name)
         group = Group.find(group_name)
         {

data/lib/pg_easy_replicate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEasyReplicate
-  VERSION = "0.1.3"
+  VERSION = "0.1.5"
 end

data/lib/pg_easy_replicate.rb

@@ -2,7 +2,6 @@
 
 require "json"
 require "ougai"
-require "lockbox"
 require "pg"
 require "sequel"
 
@@ -22,17 +21,20 @@ module PgEasyReplicate
   extend Helper
 
   class << self
-    def config
+    def config(special_user_role: nil)
       abort_with("SOURCE_DB_URL is missing") if source_db_url.nil?
       abort_with("TARGET_DB_URL is missing") if target_db_url.nil?
+
       @config ||=
         begin
           q =
             "select name, setting from pg_settings where name in  ('max_wal_senders', 'max_worker_processes', 'wal_level',  'max_replication_slots', 'max_logical_replication_workers');"
 
           {
-            source_db_is_superuser: is_super_user?(source_db_url),
-            target_db_is_superuser: is_super_user?(target_db_url),
+            source_db_is_super_user:
+              is_super_user?(source_db_url, special_user_role),
+            target_db_is_super_user:
+              is_super_user?(target_db_url, special_user_role),
             source_db:
               Query.run(
                 query: q,
@@ -51,33 +53,43 @@ module PgEasyReplicate
         end
     end
 
-    def assert_config
-      unless assert_wal_level_logical(config.dig(:source_db))
+    def assert_config(special_user_role: nil)
+      config_hash = config(special_user_role: special_user_role)
+
+      unless assert_wal_level_logical(config_hash.dig(:source_db))
         abort_with("WAL_LEVEL should be LOGICAL on source DB")
       end
 
-      unless assert_wal_level_logical(config.dig(:target_db))
+      unless assert_wal_level_logical(config_hash.dig(:target_db))
         abort_with("WAL_LEVEL should be LOGICAL on target DB")
       end
 
-      unless config.dig(:source_db_is_superuser)
-        abort_with("User on source database should be a superuser")
+      unless config_hash.dig(:source_db_is_super_user)
+        abort_with("User on source database does not have super user privilege")
       end
 
-      return if config.dig(:target_db_is_superuser)
-      abort_with("User on target database should be a superuser")
+      return if config_hash.dig(:target_db_is_super_user)
+      abort_with("User on target database does not have super user privilege")
     end
 
     def bootstrap(options)
-      assert_config
       logger.info("Setting up schema")
       setup_schema
 
       logger.info("Setting up replication user on source database")
-      create_user(conn_string: source_db_url, group_name: options[:group_name])
+      create_user(
+        conn_string: source_db_url,
+        group_name: options[:group_name],
+        special_user_role: options[:special_user_role],
+        grant_permissions_on_schema: true,
+      )
 
       logger.info("Setting up replication user on target database")
-      create_user(conn_string: target_db_url, group_name: options[:group_name])
+      create_user(
+        conn_string: target_db_url,
+        group_name: options[:group_name],
+        special_user_role: options[:special_user_role],
+      )
 
       logger.info("Setting up groups tables")
       Group.setup
@@ -123,7 +135,10 @@ module PgEasyReplicate
         query: "DROP SCHEMA IF EXISTS #{internal_schema_name} CASCADE",
         connection_url: source_db_url,
         schema: internal_schema_name,
+        user: db_user(target_db_url),
       )
+    rescue => e
+      raise "Unable to drop schema: #{e.message}"
     end
 
     def setup_schema
@@ -139,6 +154,8 @@ module PgEasyReplicate
         schema: internal_schema_name,
         user: db_user(target_db_url),
       )
+    rescue => e
+      raise "Unable to setup schema: #{e.message}"
     end
 
     def logger
@@ -160,38 +177,109 @@ module PgEasyReplicate
       end
     end
 
-    def is_super_user?(url)
-      Query.run(
-        query:
-          "select usesuper from pg_user where usename = '#{db_user(url)}';",
-        connection_url: url,
-        user: db_user(target_db_url),
-      ).first[
-        :usesuper
-      ]
+    def is_super_user?(url, special_user_role = nil)
+      if special_user_role
+        sql = <<~SQL
+          SELECT r.rolname AS username,
+            r1.rolname AS "role"
+          FROM pg_catalog.pg_roles r
+          LEFT JOIN pg_catalog.pg_auth_members m ON (m.member = r.oid)
+          LEFT JOIN pg_roles r1 ON (m.roleid=r1.oid)
+          WHERE r.rolname = '#{db_user(url)}'
+          ORDER BY 1;
+        SQL
+
+        r =
+          Query.run(
+            query: sql,
+            connection_url: url,
+            user: db_user(target_db_url),
+          )
+        # If special_user_role is passed just ensure the url in conn_string has been granted
+        # the special_user_role
+        r.any? { |q| q[:role] == special_user_role }
+      else
+        r =
+          Query.run(
+            query:
+              "SELECT rolname, rolsuper FROM pg_roles where rolname = '#{db_user(url)}';",
+            connection_url: url,
+            user: db_user(target_db_url),
+          )
+        r.any? { |q| q[:rolsuper] }
+      end
+    rescue => e
+      raise "Unable to check superuser conditions: #{e.message}"
     end
 
-    def create_user(conn_string:, group_name:)
-      password = connection_info(conn_string)[:user]
+    def create_user(
+      conn_string:,
+      group_name:,
+      special_user_role: nil,
+      grant_permissions_on_schema: false
+    )
+      password = connection_info(conn_string)[:password].gsub("'") { "''" }
+
       sql = <<~SQL
         drop role if exists #{internal_user_name};
-        create role #{internal_user_name} with password '#{password}' login superuser createdb createrole;
+        create role #{internal_user_name} with password '#{password}' login createdb createrole;
+        grant all privileges on database #{db_name(conn_string)} TO #{internal_user_name};
       SQL
 
       Query.run(
         query: sql,
         connection_url: conn_string,
-        user: db_user(target_db_url),
+        user: db_user(conn_string),
+        transaction: false,
+      )
+
+      sql =
+        if special_user_role
+          "grant #{special_user_role} to #{internal_user_name};"
+        else
+          "alter user #{internal_user_name} with superuser;"
+        end
+
+      Query.run(
+        query: sql,
+        connection_url: conn_string,
+        user: db_user(conn_string),
+        transaction: false,
       )
+
+      return unless grant_permissions_on_schema
+      Query.run(
+        query:
+          "grant all on schema #{internal_schema_name} to #{internal_user_name}",
+        connection_url: conn_string,
+        user: db_user(conn_string),
+        transaction: false,
+      )
+    rescue => e
+      raise "Unable to create user: #{e.message}"
     end
 
     def drop_user(conn_string:, group_name:)
-      sql = "drop role if exists #{internal_user_name};"
+      sql = <<~SQL
+       revoke all privileges on database #{db_name(conn_string)} from #{internal_user_name};
+      SQL
       Query.run(
         query: sql,
         connection_url: conn_string,
         user: db_user(conn_string),
       )
+
+      sql = <<~SQL
+        drop role if exists #{internal_user_name};
+      SQL
+
+      Query.run(
+        query: sql,
+        connection_url: conn_string,
+        user: db_user(conn_string),
+      )
+    rescue => e
+      raise "Unable to drop user: #{e.message}"
     end
   end
 end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: pg_easy_replicate
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-23 00:00:00.000000000 Z
+date: 2023-06-24 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: lockbox
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.2.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: ougai
   requirement: !ruby/object:Gem::Requirement
@@ -256,6 +242,7 @@ executables:
 - pg_easy_replicate
 - pg_easy_replicate_console
 - release.sh
+- test.sh
 extensions: []
 extra_rdoc_files: []
 files:
@@ -274,6 +261,7 @@ files:
 - bin/pg_easy_replicate
 - bin/pg_easy_replicate_console
 - bin/release.sh
+- bin/test.sh
 - docker-compose.yml
 - lib/pg_easy_replicate.rb
 - lib/pg_easy_replicate/cli.rb