pg_easy_replicate 0.1.5 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f655d3bf7d458de6b84a85930e7175c05f780a390e9431531a0ae30c35d4737b
-  data.tar.gz: fa23c33715823598d4795ba64cbf466088d01fd97315bfe97d3a05db2da50f48
+  metadata.gz: 8b855bf14bd72bcf4ace60b2f746139d76408d6a00e0ae7ec3fd2d0c276cf22b
+  data.tar.gz: 1eff23811d7e9d8608c241bcc7a67367c8de8271ea53d6960f57859aa5c186fd
 SHA512:
-  metadata.gz: 1e69d89ca00ca077784aab0c042e46c85faf7e15f2e585a65321084c5b3543502a7c9a440faa17f829848015a31ed50393129a0d7bfa083a27a731af4c179b68
-  data.tar.gz: ce29729ddcf74420818c0b4d86b98de28fbefd0690368ee4fcdd40f689dcf1ba6c3135850ab1e22de2bbd977c733ca5449cf42ccef728fda20c54b102b38f965
+  metadata.gz: a7de21167ee2027115972442b9aa859fba7954bdc2e04b8e27e92d6fbadd2e7c9eb92c700d01f69ba9fecd5b40fe5593b5d0467ba4b2766106a7951311401842
+  data.tar.gz: c38c45df66c9a4433b558a41f120f8c3304b5cbd9ea1072348df61397b7e51b38d1709b83c160917d69fbcf460e18dc9f9355ea7c96ae3d109ba40b4430ff06c

data/.rspec

@@ -2,4 +2,4 @@
 --color
 --require spec_helper
 --fail-fast
---exclude-pattern spec/lib/smoke_spec.rb
+--exclude-pattern spec/smoke_spec.rb

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## [0.1.6] - 2023-06-22
+
+- Bug fix: Support custom schema name
+- New smoke spec in CI
+
+## [0.1.5] - 2023-06-22
+
+- Fix bug in `stop_sync`
+
 ## [0.1.4] - 2023-06-22
 
 - Drop lockbox dependency

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pg_easy_replicate (0.1.5)
+    pg_easy_replicate (0.1.7)
       ougai (~> 2.0.0)
       pg (~> 1.5.3)
       sequel (~> 5.69.0)

data/README.md

@@ -1,7 +1,8 @@
 # pg_easy_replicate
 
 [![CI](https://github.com/shayonj/pg_easy_replicate/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg_easy_replicate/actions/workflows/ci.yaml)
-[![Gem Version](https://badge.fury.io/rb/pg_easy_replicate.svg)](https://badge.fury.io/rb/pg_easy_replicate)
+[![Smoke spec](https://github.com/shayonj/pg_easy_replicate/actions/workflows/smoke.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg_easy_replicate/actions/workflows/ci.yaml)
+[![Gem Version](https://badge.fury.io/rb/pg_easy_replicate.svg?1)](https://badge.fury.io/rb/pg_easy_replicate)
 
 `pg_easy_replicate` is a CLI orchestrator tool that simplifies the process of setting up [logical replication](https://www.postgresql.org/docs/current/logical-replication.html) between two PostgreSQL databases. `pg_easy_replicate` also supports switchover. After the source (primary database) is fully replicating, `pg_easy_replicate` puts it into read-only mode and via logical replication flushes all data to the new target database. This ensures zero data loss and minimal downtime for the application. This method can be useful for performing minimal downtime (up to <1min, depending) major version upgrades between two PostgreSQL databases, load testing with blue/green database setup and other similar use cases.
 

data/lib/pg_easy_replicate/orchestrate.rb

@@ -9,6 +9,14 @@ module PgEasyReplicate
       DEFAULT_WAIT = 5 # seconds
 
       def start_sync(options)
+        schema_name = options[:schema_name] || "public"
+        tables =
+          determine_tables(
+            schema: schema_name,
+            conn_string: source_db_url,
+            list: options[:tables],
+          )
+
         create_publication(
           group_name: options[:group_name],
           conn_string: source_db_url,
@@ -16,9 +24,9 @@ module PgEasyReplicate
 
         add_tables_to_publication(
           group_name: options[:group_name],
-          tables: options[:tables],
+          tables: tables,
           conn_string: source_db_url,
-          schema: options[:schema],
+          schema: schema_name,
         )
 
         create_subscription(
@@ -29,8 +37,8 @@ module PgEasyReplicate
 
         Group.create(
           name: options[:group_name],
-          table_names: options[:tables],
-          schema_name: options[:schema],
+          table_names: tables,
+          schema_name: schema_name,
           started_at: Time.now.utc,
         )
       rescue => e
@@ -45,8 +53,8 @@ module PgEasyReplicate
         else
           Group.create(
             name: options[:group_name],
-            table_names: options[:tables],
-            schema_name: options[:schema],
+            table_names: tables,
+            schema_name: schema_name,
             started_at: Time.now.utc,
             failed_at: Time.now.utc,
           )
@@ -80,19 +88,17 @@ module PgEasyReplicate
           "Adding tables up publication",
           { publication_name: publication_name(group_name) },
         )
-        tables = tables&.split(",") || []
-        unless tables.size > 0
-          tables = list_all_tables(schema: schema, conn_string: conn_string)
-        end
 
-        tables.map do |table_name|
-          Query.run(
-            query:
-              "ALTER PUBLICATION #{publication_name(group_name)} ADD TABLE \"#{table_name}\"",
-            connection_url: conn_string,
-            schema: schema,
-          )
-        end
+        tables
+          .split(",")
+          .map do |table_name|
+            Query.run(
+              query:
+                "ALTER PUBLICATION #{publication_name(group_name)} ADD TABLE \"#{table_name}\"",
+              connection_url: conn_string,
+              schema: schema,
+            )
+          end
       rescue => e
         raise "Unable to add tables to publication: #{e.message}"
       end
@@ -101,11 +107,12 @@ module PgEasyReplicate
         Query
           .run(
             query:
-              "SELECT table_name FROM information_schema.tables WHERE table_schema = '#{schema}'",
+              "SELECT table_name FROM information_schema.tables WHERE table_schema = '#{schema}' ORDER BY table_name",
             connection_url: conn_string,
           )
           .map(&:values)
           .flatten
+          .join(",")
       end
 
       def drop_publication(group_name:, conn_string:)
@@ -197,11 +204,16 @@ module PgEasyReplicate
         group_name:,
         source_conn_string: source_db_url,
         target_conn_string: target_db_url,
-        lag_delta_size: DEFAULT_LAG
+        lag_delta_size: nil
       )
         group = Group.find(group_name)
 
-        watch_lag(group_name: group_name, lag: lag_delta_size)
+        run_vacuum_analyze(
+          conn_string: target_conn_string,
+          tables: group[:table_names],
+          schema: group[:schema_name],
+        )
+        watch_lag(group_name: group_name, lag: lag_delta_size || DEFAULT_LAG)
         revoke_connections_on_source_db(group_name)
         wait_for_remaining_catchup(group_name)
         refresh_sequences(
@@ -209,6 +221,12 @@ module PgEasyReplicate
           schema: group[:schema_name],
         )
         mark_switchover_complete(group_name)
+        # Run vacuum analyze to refresh the planner post switchover
+        run_vacuum_analyze(
+          conn_string: target_conn_string,
+          tables: group[:table_names],
+          schema: group[:schema_name],
+        )
         drop_subscription(
           group_name: group_name,
           target_conn_string: target_conn_string,
@@ -317,9 +335,39 @@ module PgEasyReplicate
         raise "Unable to refresh sequences: #{e.message}"
       end
 
+      def run_vacuum_analyze(conn_string:, tables:, schema:)
+        tables
+          .split(",")
+          .each do |t|
+            logger.info(
+              "Running vacuum analyze on #{t}",
+              schema: schema,
+              table: t,
+            )
+            Query.run(
+              query: "VACUUM VERBOSE ANALYZE #{t}",
+              connection_url: conn_string,
+              schema: schema,
+              transaction: false,
+            )
+          end
+      rescue => e
+        raise "Unable to run vacuum and analyze: #{e.message}"
+      end
+
       def mark_switchover_complete(group_name)
         Group.update(group_name: group_name, switchover_completed_at: Time.now)
       end
+
+      private
+
+      def determine_tables(schema:, conn_string:, list: "")
+        tables = list&.split(",") || []
+        unless tables.size > 0
+          return list_all_tables(schema: schema, conn_string: conn_string)
+        end
+        ""
+      end
     end
   end
 end

data/lib/pg_easy_replicate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEasyReplicate
-  VERSION = "0.1.5"
+  VERSION = "0.1.7"
 end

data/scripts/e2e-bootstrap.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eo pipefail
+
+if [[ -z ${GITHUB_WORKFLOW} ]]; then
+  export SECONDARY_SOURCE_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@source_db/postgres"
+fi
+
+export SOURCE_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@localhost:5432/postgres"
+export TARGET_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@localhost:5433/postgres"
+export PGPASSWORD='jamesbond123@7!'"'"'3aaR'
+
+pgbench --initialize -s 5 --foreign-keys --host localhost -U jamesbond -d postgres
+
+pg_dump --schema-only --host localhost -U jamesbond -d postgres >schema.sql
+cat schema.sql | psql --host localhost -U jamesbond -d postgres -p 5433
+rm schema.sql
+
+bundle exec bin/pg_easy_replicate config_check

data/scripts/e2e-start.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -eo pipefail
+
+if [[ -z ${GITHUB_WORKFLOW} ]]; then
+  export SECONDARY_SOURCE_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@source_db/postgres"
+fi
+
+export SOURCE_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@localhost:5432/postgres"
+export TARGET_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@localhost:5433/postgres"
+export PGPASSWORD='jamesbond123@7!'"'"''"'"'3aaR'
+
+# Bootstrap and cleanup
+echo "===== Performing Bootstrap and cleanup"
+bundle exec bin/pg_easy_replicate bootstrap -g cluster-1
+bundle exec bin/pg_easy_replicate start_sync -g cluster-1 -s public
+bundle exec bin/pg_easy_replicate stats -g cluster-1
+bundle exec bin/pg_easy_replicate switchover -g cluster-1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_easy_replicate
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.7
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-24 00:00:00.000000000 Z
+date: 2023-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai
@@ -241,8 +241,6 @@ email:
 executables:
 - pg_easy_replicate
 - pg_easy_replicate_console
-- release.sh
-- test.sh
 extensions: []
 extra_rdoc_files: []
 files:
@@ -260,8 +258,6 @@ files:
 - Rakefile
 - bin/pg_easy_replicate
 - bin/pg_easy_replicate_console
-- bin/release.sh
-- bin/test.sh
 - docker-compose.yml
 - lib/pg_easy_replicate.rb
 - lib/pg_easy_replicate/cli.rb
@@ -272,6 +268,9 @@ files:
 - lib/pg_easy_replicate/stats.rb
 - lib/pg_easy_replicate/version.rb
 - package.json
+- scripts/e2e-bootstrap.sh
+- scripts/e2e-start.sh
+- scripts/release.sh
 - yarn.lock
 homepage: https://github.com/shayonj/pg_easy_replicate
 licenses:

data/bin/test.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-export SECONDARY_SOURCE_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@source_db/postgres"
-export SOURCE_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@localhost:5432/postgres"
-export TARGET_DB_URL="postgres://jamesbond:jamesbond123%407%21%273aaR@localhost:5433/postgres"
-
-bundle exec bin/pg_easy_replicate config_check
-
-# Bootstrap and cleanup
-echo "===== Performing Bootstrap and cleanup"
-bundle exec bin/pg_easy_replicate bootstrap -g cluster-1
-bundle exec bin/pg_easy_replicate cleanup -e -g cluster-1
-
-# Bootstrap and start_sync
-echo "===== Performing Bootstrap, start_sync, stop_sync and cleanup"
-bundle exec bin/pg_easy_replicate bootstrap -g cluster-1
-bundle exec bin/pg_easy_replicate start_sync -g cluster-1
-bundle exec bin/pg_easy_replicate stop_sync -g cluster-1
-bundle exec bin/pg_easy_replicate cleanup -e -g cluster-1
-
-# Bootstrap with switchover
-echo "===== Performing Bootstrap, start_sync, stop_sync and cleanup"
-bundle exec bin/pg_easy_replicate bootstrap -g cluster-1
-bundle exec bin/pg_easy_replicate start_sync -g cluster-1
-# bundle exec bin/pg_easy_replicate switchover -g cluster-1
-bundle exec bin/pg_easy_replicate cleanup -e -g cluster-1