pg_easy_replicate 0.1.2 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5df08d2cfc5874389ab9a5311a572a39ea76e5c2a56a53df3db935865caeb99
-  data.tar.gz: 156656c80d4d808bc1a623507e0a2da904f9c7ebd26ec9e767382aac90276d70
+  metadata.gz: c253c57988226f7d90116cfc4276a51ac6a2753daea88169ba244fe4ce069e76
+  data.tar.gz: cadca2e9e430f06be91ce4e1623fdf333fdc96edded99e83729f7b56e9356191
 SHA512:
-  metadata.gz: 47e224f6e9d1133f5a2eee992e42afddefdf89f908dc763c84d4e36547849f3e69824f47481df05d2552d7708240af75aa32f056e2cacf60293fdefe983fc2ed
-  data.tar.gz: '0184bfa3eafd4224cc5eefa32f45fead0491e507e7df78e0395a06dd9ce22bd4b76ea59e4d253113e0fc3dbdbb73b548175d44f87a4acf070633657629310568'
+  metadata.gz: dee2a8f5c96289076276b6d2f92e5e0623ac1c2ed7d885bfa7ab472bbc80fc525665fa72ed54fb772d63e9fd564acecea5c0a3d107dcb5e46370e539c20ab2ac
+  data.tar.gz: b7c9c94cf5c5b9288a81e6d2a144f71cc387096b464f732f38755f293c1d7344120cf4107f1803981524773edd95cabfa34c56b973004dfd0efec69e9f066f8a

data/.ruby-version

@@ -1 +1 @@
-3.1.3
+3.1.4

data/Dockerfile

@@ -1,5 +1,5 @@
-FROM ruby:3.0
+FROM ruby:3.1.4
 
-ARG VERSION=0.1.2
+ARG VERSION
 
 RUN gem install pg_easy_replicate -v $VERSION

data/Gemfile.lock

@@ -1,8 +1,7 @@
 PATH
   remote: .
   specs:
-    pg_easy_replicate (0.1.2)
-      lockbox (~> 1.2.0)
+    pg_easy_replicate (0.1.4)
       ougai (~> 2.0.0)
       pg (~> 1.5.3)
       sequel (~> 5.69.0)
@@ -19,7 +18,6 @@ GEM
       thor
       tilt
     json (2.6.3)
-    lockbox (1.2.0)
     method_source (1.0.0)
     oj (3.14.3)
     ougai (2.0.0)

data/README.md

@@ -15,6 +15,9 @@ Battle tested in production at [Tines](https://www.tines.com/) 🚀
 - [Replicating all tables with a single group](#replicating-all-tables-with-a-single-group)
   - [Config check](#config-check)
   - [Bootstrap](#bootstrap)
+  - [Bootstrap and Config Check with special user role (AWS/GCP/Custom)](#bootstrap-and-config-check-with-special-user-role--aws-gcp-custom-)
+    - [Config Check](#config-check)
+    - [Bootstrap](#bootstrap-1)
   - [Start sync](#start-sync)
   - [Stats](#stats)
   - [Performing switchover](#performing-switchover)
@@ -22,6 +25,7 @@ Battle tested in production at [Tines](https://www.tines.com/) 🚀
 - [Switchover strategies with minimal downtime](#switchover-strategies-with-minimal-downtime)
   - [Rolling restart strategy](#rolling-restart-strategy)
   - [DNS Failover strategy](#dns-failover-strategy)
+- [Contributing](#contributing)
 
 ## Installation
 
@@ -51,7 +55,7 @@ https://hub.docker.com/r/shayonj/pg_easy_replicate
 
 - PostgreSQL 10 and later
 - Ruby 2.7 and later
-- Database user should have permissions for `SUPERUSER`
+- Database user should have permissions for `SUPERUSER` or pass in the special user role that has the privileges to create role, schema, publication and subscription on both databases. More on `--special-user-role` section below.
 - Both databases should have the same schema
 
 ## Limits
@@ -116,6 +120,31 @@ $ pg_easy_replicate bootstrap --group-name database-cluster-1
 ...
 ```
 
+### Bootstrap and Config Check with special user role (AWS/GCP/Custom)
+
+If you don't want your primary login user to have `superuser` privileges or you are on AWS or GCP, you will need to pass in the special user role that has the privileges to create role, schema, publication and subscription. This is required so `pg_easy_replicate` can create a dedicated user for replication which is granted the respective special user role to carry out its functionalities.
+
+For AWS the special user role is `rds_superuser`, and for GCP it is `cloudsqlsuperuser`. Please refer to docs for the most up to date information.
+
+**Note**: The user in the connection url must be part of the special user role being supplied.
+
+#### Config Check
+
+```bash
+$ pg_easy_replicate config_check --special-user-role="rds_superuser"
+
+✅ Config is looking good.
+```
+
+#### Bootstrap
+
+```bash
+$ pg_easy_replicate bootstrap --group-name database-cluster-1 --special-user-role="rds_superuser"
+
+{"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":21485,"level":30,"time":"2023-06-19T15:51:11.015-04:00","v":0,"msg":"Setting up schema","version":"0.1.0"}
+...
+```
+
 ### Start sync
 
 Once the bootstrap is complete, you can start the sync. Starting the sync sets up the publication, subscription and performs other minor housekeeping things.
@@ -216,3 +245,11 @@ Next, you can set up a program that watches the `stats` and waits until `switcho
 In this strategy, you have a weighted based DNS system (example [AWS Route53 weighted records](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/resource-record-sets-values-weighted.html)) where 100% of traffic goes to a primary origin and 0% to a secondary origin. The primary origin here is the DNS host for your source database and secondary origin is the DNS host for your target database. You can set up your application ahead of time to interact with the database using DNS from the weighted group.
 
 Next, you can set up a program that watches the `stats` and waits until `switchover_completed_at` is reporting as `true`. Once that happens it updates the weight in the DNS weighted group where 100% of the requests now go to the new/target database. Note: Keeping a low `ttl` is recommended.
+
+## Contributing
+
+PRs most welcome. You can get started locally by
+
+- `docker compose down -v && docker compose up --remove-orphans --build`
+- Install ruby `3.1.4` using RVM ([instruction](https://rvm.io/rvm/install#any-other-system))
+- `bundle exec rspec` for specs

data/bin/release.sh

@@ -1,28 +1,18 @@
 export VERSION=$1
 echo "VERSION: ${VERSION}"
 
-# echo "=== Pushing tags to github ===="
-# git tag v"$VERSION"
-# git push origin --tags
-
 echo "=== Building Gem ===="
 gem build pg_easy_replicate.gemspec
 
 echo "=== Pushing gem ===="
 gem push pg_easy_replicate-"$VERSION".gem
 
-echo "=== Sleeping for 5s ===="
-sleep 5
-
-echo "=== Building Image ===="
-docker build . --build-arg VERSION="$VERSION" -t shayonj/pg_easy_replicate:"$VERSION"
-
-echo "=== Tagging Image ===="
-docker image tag shayonj/pg_easy_replicate:"$VERSION" shayonj/pg_easy_replicate:latest
+echo "=== Sleeping for 15s ===="
+sleep 15
 
-echo "=== Pushing Image ===="
-docker push shayonj/pg_easy_replicate:"$VERSION"
-docker push shayonj/pg_easy_replicate:latest
+echo "=== Pushing tags to github ===="
+git tag v"$VERSION"
+git push origin --tags
 
 echo "=== Cleaning up ===="
 rm pg_easy_replicate-"$VERSION".gem

data/docker-compose.yml

@@ -1,32 +1,34 @@
 version: "3.7"
 services:
   source_db:
-    image: postgres:12-alpine
+    image: postgres:12
     ports:
       - "5432:5432"
     environment:
       POSTGRES_USER: jamesbond
-      POSTGRES_PASSWORD: jamesbond
+      POSTGRES_PASSWORD: jamesbond123@7!'3aaR
       POSTGRES_DB: postgres
-    command:
-      - "postgres"
-      - "-c"
-      - "wal_level=logical"
+    command: >
+      -c wal_level=logical
+      -c ssl=on
+      -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
+      -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
     networks:
       localnet:
 
   target_db:
-    image: postgres:12-alpine
+    image: postgres:12
     ports:
       - "5433:5432"
     environment:
       POSTGRES_USER: jamesbond
-      POSTGRES_PASSWORD: jamesbond
+      POSTGRES_PASSWORD: jamesbond123@7!'3aaR
       POSTGRES_DB: postgres
-    command:
-      - "postgres"
-      - "-c"
-      - "wal_level=logical"
+    command: >
+      -c wal_level=logical
+      -c ssl=on
+      -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
+      -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
     networks:
       localnet:
 

data/lib/pg_easy_replicate/cli.rb

@@ -8,8 +8,14 @@ module PgEasyReplicate
 
     desc "config_check",
          "Prints if source and target database have the required config"
-    def config_check
-      PgEasyReplicate.assert_config
+    method_option :special_user_role,
+                  aliases: "-s",
+                  desc:
+                    "Name of the role that has superuser permissions. Usually useful for AWS (rds_superuser) or GCP (cloudsqlsuperuser)."
+    def config_check(options)
+      PgEasyReplicate.assert_config(
+        special_user_role: options[:special_user_role],
+      )
 
       puts "✅ Config is looking good."
     end
@@ -18,6 +24,10 @@ module PgEasyReplicate
                   aliases: "-g",
                   required: true,
                   desc: "Name of the group to provision"
+    method_option :special_user_role,
+                  aliases: "-s",
+                  desc:
+                    "Name of the role that has superuser permissions. Usually useful with AWS (rds_superuser) or GCP (cloudsqlsuperuser)."
     desc "bootstrap",
          "Sets up temporary tables for information required during runtime"
     def bootstrap

data/lib/pg_easy_replicate/helper.rb

@@ -60,6 +60,10 @@ module PgEasyReplicate
       connection_info(url)[:user]
     end
 
+    def db_name(url)
+      connection_info(url)[:dbname]
+    end
+
     def abort_with(msg)
       raise(msg) if test_env?
       abort(msg)

data/lib/pg_easy_replicate/orchestrate.rb

@@ -9,8 +9,6 @@ module PgEasyReplicate
       DEFAULT_WAIT = 5 # seconds
 
       def start_sync(options)
-        PgEasyReplicate.assert_config
-
         create_publication(
           group_name: options[:group_name],
           conn_string: source_db_url,
@@ -62,10 +60,14 @@ module PgEasyReplicate
           "Setting up publication",
           { publication_name: publication_name(group_name) },
         )
+
         Query.run(
           query: "create publication #{publication_name(group_name)}",
           connection_url: conn_string,
+          user: db_user(conn_string),
         )
+      rescue => e
+        raise "Unable to create publication: #{e.message}"
       end
 
       def add_tables_to_publication(
@@ -91,6 +93,8 @@ module PgEasyReplicate
             schema: schema,
           )
         end
+      rescue => e
+        raise "Unable to add tables to publication: #{e.message}"
       end
 
       def list_all_tables(schema:, conn_string:)
@@ -112,7 +116,10 @@ module PgEasyReplicate
         Query.run(
           query: "DROP PUBLICATION IF EXISTS #{publication_name(group_name)}",
           connection_url: conn_string,
+          user: db_user(conn_string),
         )
+      rescue => e
+        raise "Unable to drop publication: #{e.message}"
       end
 
       def create_subscription(
@@ -132,6 +139,7 @@ module PgEasyReplicate
           query:
             "CREATE SUBSCRIPTION #{subscription_name(group_name)} CONNECTION '#{source_conn_string}' PUBLICATION #{publication_name(group_name)}",
           connection_url: target_conn_string,
+          user: db_user(target_conn_string),
           transaction: false,
         )
       rescue Sequel::DatabaseError => e
@@ -141,7 +149,7 @@ module PgEasyReplicate
           )
         end
 
-        raise
+        raise "Unable to create subscription: #{e.message}"
       end
 
       def drop_subscription(group_name:, target_conn_string:)
@@ -157,11 +165,11 @@ module PgEasyReplicate
           connection_url: target_conn_string,
           transaction: false,
         )
+      rescue => e
+        raise "Unable to drop subscription: #{e.message}"
       end
 
       def stop_sync(target_conn_string:, source_conn_string:, group_name:)
-        PgEasyReplicate.assert_config
-
         logger.info(
           "Stopping sync",
           {
@@ -177,6 +185,8 @@ module PgEasyReplicate
           group_name: group_name,
           target_conn_string: target_conn_string,
         )
+      rescue => e
+        raise "Unable to stop sync user: #{e.message}"
       end
 
       def switchover(
@@ -185,7 +195,6 @@ module PgEasyReplicate
         target_conn_string: target_db_url,
         lag_delta_size: DEFAULT_LAG
       )
-        PgEasyReplicate.assert_config
         group = Group.find(group_name)
 
         watch_lag(group_name: group_name, lag: lag_delta_size)
@@ -258,6 +267,8 @@ module PgEasyReplicate
           "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE usename = '#{db_user(source_db_url)}';"
 
         Query.run(query: kill_sql, connection_url: source_db_url)
+      rescue => e
+        raise "Unable to revoke connections on source db: #{e.message}"
       end
 
       def restore_connections_on_source_db(group_name)
@@ -298,6 +309,8 @@ module PgEasyReplicate
         SQL
 
         Query.run(query: sql, connection_url: conn_string, schema: schema)
+      rescue => e
+        raise "Unable to refresh sequences: #{e.message}"
       end
 
       def mark_switchover_complete(group_name)

data/lib/pg_easy_replicate/stats.rb

@@ -13,7 +13,6 @@ module PgEasyReplicate
 
     class << self
       def object(group_name)
-        PgEasyReplicate.assert_config
         stats = replication_stats(group_name)
         group = Group.find(group_name)
         {

data/lib/pg_easy_replicate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEasyReplicate
-  VERSION = "0.1.2"
+  VERSION = "0.1.4"
 end

data/lib/pg_easy_replicate.rb

@@ -2,7 +2,6 @@
 
 require "json"
 require "ougai"
-require "lockbox"
 require "pg"
 require "sequel"
 
@@ -22,17 +21,20 @@ module PgEasyReplicate
   extend Helper
 
   class << self
-    def config
+    def config(special_user_role: nil)
       abort_with("SOURCE_DB_URL is missing") if source_db_url.nil?
       abort_with("TARGET_DB_URL is missing") if target_db_url.nil?
+
       @config ||=
         begin
           q =
             "select name, setting from pg_settings where name in  ('max_wal_senders', 'max_worker_processes', 'wal_level',  'max_replication_slots', 'max_logical_replication_workers');"
 
           {
-            source_db_is_superuser: is_super_user?(source_db_url),
-            target_db_is_superuser: is_super_user?(target_db_url),
+            source_db_is_super_user:
+              is_super_user?(source_db_url, special_user_role),
+            target_db_is_super_user:
+              is_super_user?(target_db_url, special_user_role),
             source_db:
               Query.run(
                 query: q,
@@ -51,33 +53,43 @@ module PgEasyReplicate
         end
     end
 
-    def assert_config
-      unless assert_wal_level_logical(config.dig(:source_db))
+    def assert_config(special_user_role: nil)
+      config_hash = config(special_user_role: special_user_role)
+
+      unless assert_wal_level_logical(config_hash.dig(:source_db))
         abort_with("WAL_LEVEL should be LOGICAL on source DB")
       end
 
-      unless assert_wal_level_logical(config.dig(:target_db))
+      unless assert_wal_level_logical(config_hash.dig(:target_db))
         abort_with("WAL_LEVEL should be LOGICAL on target DB")
       end
 
-      unless config.dig(:source_db_is_superuser)
-        abort_with("User on source database should be a superuser")
+      unless config_hash.dig(:source_db_is_super_user)
+        abort_with("User on source database does not have super user privilege")
       end
 
-      return if config.dig(:target_db_is_superuser)
-      abort_with("User on target database should be a superuser")
+      return if config_hash.dig(:target_db_is_super_user)
+      abort_with("User on target database does not have super user privilege")
     end
 
     def bootstrap(options)
-      assert_config
       logger.info("Setting up schema")
       setup_schema
 
       logger.info("Setting up replication user on source database")
-      create_user(conn_string: source_db_url, group_name: options[:group_name])
+      create_user(
+        conn_string: source_db_url,
+        group_name: options[:group_name],
+        special_user_role: options[:special_user_role],
+        grant_permissions_on_schema: true,
+      )
 
       logger.info("Setting up replication user on target database")
-      create_user(conn_string: target_db_url, group_name: options[:group_name])
+      create_user(
+        conn_string: target_db_url,
+        group_name: options[:group_name],
+        special_user_role: options[:special_user_role],
+      )
 
       logger.info("Setting up groups tables")
       Group.setup
@@ -123,7 +135,10 @@ module PgEasyReplicate
         query: "DROP SCHEMA IF EXISTS #{internal_schema_name} CASCADE",
         connection_url: source_db_url,
         schema: internal_schema_name,
+        user: db_user(target_db_url),
       )
+    rescue => e
+      raise "Unable to drop schema: #{e.message}"
     end
 
     def setup_schema
@@ -139,6 +154,8 @@ module PgEasyReplicate
         schema: internal_schema_name,
         user: db_user(target_db_url),
       )
+    rescue => e
+      raise "Unable to setup schema: #{e.message}"
     end
 
     def logger
@@ -160,38 +177,109 @@ module PgEasyReplicate
       end
     end
 
-    def is_super_user?(url)
-      Query.run(
-        query:
-          "select usesuper from pg_user where usename = '#{db_user(url)}';",
-        connection_url: url,
-        user: db_user(target_db_url),
-      ).first[
-        :usesuper
-      ]
+    def is_super_user?(url, special_user_role = nil)
+      if special_user_role
+        sql = <<~SQL
+          SELECT r.rolname AS username,
+            r1.rolname AS "role"
+          FROM pg_catalog.pg_roles r
+          LEFT JOIN pg_catalog.pg_auth_members m ON (m.member = r.oid)
+          LEFT JOIN pg_roles r1 ON (m.roleid=r1.oid)
+          WHERE r.rolname = '#{db_user(url)}'
+          ORDER BY 1;
+        SQL
+
+        r =
+          Query.run(
+            query: sql,
+            connection_url: url,
+            user: db_user(target_db_url),
+          )
+        # If special_user_role is passed just ensure the url in conn_string has been granted
+        # the special_user_role
+        r.any? { |q| q[:role] == special_user_role }
+      else
+        r =
+          Query.run(
+            query:
+              "SELECT rolname, rolsuper FROM pg_roles where rolname = '#{db_user(url)}';",
+            connection_url: url,
+            user: db_user(target_db_url),
+          )
+        r.any? { |q| q[:rolsuper] }
+      end
+    rescue => e
+      raise "Unable to check superuser conditions: #{e.message}"
     end
 
-    def create_user(conn_string:, group_name:)
-      password = connection_info(conn_string)[:user]
+    def create_user(
+      conn_string:,
+      group_name:,
+      special_user_role: nil,
+      grant_permissions_on_schema: false
+    )
+      password = connection_info(conn_string)[:password].gsub("'") { "''" }
+
       sql = <<~SQL
         drop role if exists #{internal_user_name};
-        create role #{internal_user_name} with password '#{password}' login superuser createdb createrole;
+        create role #{internal_user_name} with password '#{password}' login createdb createrole;
+        grant all privileges on database #{db_name(conn_string)} TO #{internal_user_name};
       SQL
 
       Query.run(
         query: sql,
         connection_url: conn_string,
-        user: db_user(target_db_url),
+        user: db_user(conn_string),
+        transaction: false,
+      )
+
+      sql =
+        if special_user_role
+          "grant #{special_user_role} to #{internal_user_name};"
+        else
+          "alter user #{internal_user_name} with superuser;"
+        end
+
+      Query.run(
+        query: sql,
+        connection_url: conn_string,
+        user: db_user(conn_string),
+        transaction: false,
       )
+
+      return unless grant_permissions_on_schema
+      Query.run(
+        query:
+          "grant all on schema #{internal_schema_name} to #{internal_user_name}",
+        connection_url: conn_string,
+        user: db_user(conn_string),
+        transaction: false,
+      )
+    rescue => e
+      raise "Unable to create user: #{e.message}"
     end
 
     def drop_user(conn_string:, group_name:)
-      sql = "drop role if exists #{internal_user_name};"
+      sql = <<~SQL
+       revoke all privileges on database #{db_name(conn_string)} from #{internal_user_name};
+      SQL
       Query.run(
         query: sql,
         connection_url: conn_string,
         user: db_user(conn_string),
       )
+
+      sql = <<~SQL
+        drop role if exists #{internal_user_name};
+      SQL
+
+      Query.run(
+        query: sql,
+        connection_url: conn_string,
+        user: db_user(conn_string),
+      )
+    rescue => e
+      raise "Unable to drop user: #{e.message}"
     end
   end
 end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: pg_easy_replicate
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.4
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-22 00:00:00.000000000 Z
+date: 2023-06-24 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: lockbox
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.2.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: ougai
   requirement: !ruby/object:Gem::Requirement