pg_easy_replicate 0.1.12 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d7321dd7ea830328c42f49d3e12c14df7f2b7d9cf1448fd61a46b2236847239
-  data.tar.gz: d3aad658eab7c86766c4528c89bcd8718376504d97ca7d1cb356f7b7faa116d2
+  metadata.gz: da825eb796e5b827c9197440301d841f725a5bbaeac74b066fea67b5bbf039ac
+  data.tar.gz: 68da98e57eab118b7aea8f4e83171eaa8b18bd66a86bd5f39fb901f5aef9bfda
 SHA512:
-  metadata.gz: 7cd29a4307b130f291007dec2f9a9e5abb5227cd892349181391fce74d9c032dc9c823e44e90ba07959d3630c191f16968d6442689849199e7f98268dc5e0244
-  data.tar.gz: efff8b0ea2b1c9890a70306e83d75c6e35fa78645658d914c92a8142842834fd731cd2b707ab150e4ef043f7d3e85291eaa320b14db56a8f0867b8db94824b67
+  metadata.gz: e04ca73e6f6b63f68a37389cb788bc7a6fddd80aa75fa433d297d4a600b1929ee06341e456b8b2d12a0cb03952824c02dca63033f40675948c250bcba2849b2a
+  data.tar.gz: b0617c3d5d2defeab8f576eb6cceb1845c8c7fe7478a064f2109fbd4e63ebcf58867bd2557221810c53778e62806bacd65e92470fd368f96c5d38405bee4efec

data/.rubocop.yml

@@ -15,7 +15,7 @@ AllCops:
     - "**/.git/**/*"
     - "**/node_modules/**/*"
     - "**/Brewfile"
-  TargetRubyVersion: 2.7
+  TargetRubyVersion: 3.0
 
 Bundler/OrderedGems:
   Include:

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## [0.1.12] - 2023-12-13
+
+- Bump rubocop-rspec from 2.24.1 to 2.25.0 - #65
+- Quote indent DB name - #76
+
+## [0.1.12] - 2023-12-13
+
+- Drop existing user with privileges when bootstrapping - #75
+
 ## [0.1.10] - 2023-12-12
 
 - Reference the passed in URL and use source db url - #74

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    pg_easy_replicate (0.1.12)
+    pg_easy_replicate (0.2.0)
       ougai (~> 2.0.0)
       pg (~> 1.5.3)
       sequel (>= 5.69, < 5.76)
-      thor (~> 1.2.2)
+      thor (>= 1.2.2, < 1.4.0)
 
 GEM
   remote: https://rubygems.org/
@@ -35,7 +35,7 @@ GEM
       method_source (~> 1.0)
     racc (1.7.3)
     rainbow (3.1.1)
-    rake (13.0.6)
+    rake (13.1.0)
     rbs (3.1.0)
     regexp_parser (2.8.2)
     rexml (3.2.6)
@@ -94,7 +94,7 @@ GEM
       rbs
       syntax_tree (>= 2.0.1)
     temple (0.10.1)
-    thor (1.2.2)
+    thor (1.3.0)
     tilt (2.1.0)
     unicode-display_width (2.5.0)
 

data/README.md

@@ -8,6 +8,8 @@
 
 Battle tested in production at [Tines](https://www.tines.com/) 🚀
 
+![](./assets/mascot.png)
+
 - [Installation](#installation)
 - [Requirements](#requirements)
 - [Limits](#limits)
@@ -57,7 +59,7 @@ https://hub.docker.com/r/shayonj/pg_easy_replicate
 ## Requirements
 
 - PostgreSQL 10 and later
-- Ruby 2.7 and later
+- Ruby 3.0 and later
 - Database users should have `SUPERUSER` permissions, or pass in a special user with privileges to create the needed role, schema, publication and subscription on both databases. More on `--special-user-role` section below.
 
 ## Limits
@@ -151,6 +153,8 @@ $ pg_easy_replicate bootstrap --group-name database-cluster-1 --special-user-rol
 
 Once the bootstrap is complete, you can start the sync. Starting the sync sets up the publication, subscription and performs other minor housekeeping things.
 
+**NOTE**: Start sync by default will drop all indices in the target database for performance reasons. And will automatically re-add the indices during `switchover`. It is turned on by default and you can opt out of this with `--no-recreate-indices-post-copy`
+
 ```bash
 $ pg_easy_replicate start_sync --group-name database-cluster-1
 
@@ -203,7 +207,9 @@ $ pg_easy_replicate stats --group-name database-cluster-1
 
 `switchover` will wait until all tables in the group are replicating and the delta for lag is <200kb (by calculating the `pg_wal_lsn_diff` between `sent_lsn` and `write_lsn`) and then perform the switch.
 
-The switch is made by putting the user on the source database in `READ ONLY` mode, so that it is not accepting any more writes and waits for the flush lag to be `0`. It’s up to the user to kick off a rolling restart of your application containers or failover DNS (more on these below in strategies) after the switchover is complete, so that your application isn't sending any read/write requests to the old/source database.
+Additionally, `switchover` will take care of re-adding the indices (it had removed in `start_sync`) in the target database before hand. Depending on the size of the tables, the recreation of indexes (which happens `CONCURRENTLY`) may take a while. See `start_sync` for more details.
+
+The switch is made by putting the user on the source database in `READ ONLY` mode, so that it is not accepting any more writes and waits for the flush lag to be `0`. It’s up to the user to kick off a rolling restart of their application containers or failover DNS (more on these below in strategies) after the switchover is complete, so that your application isn't sending any read + write requests to the old/source database.
 
 ```bash
 $ pg_easy_replicate switchover  --group-name database-cluster-1

data/lib/pg_easy_replicate/cli.rb

@@ -79,6 +79,12 @@ module PgEasyReplicate
                   aliases: "-t",
                   desc:
                     "Comma separated list of table names. Default: All tables"
+    method_option :recreate_indices_post_copy,
+                  type: :boolean,
+                  default: true,
+                  aliases: "-r",
+                  desc:
+                    "Drop all non-primary indices before copy and recreate them post-copy"
     def start_sync
       PgEasyReplicate::Orchestrate.start_sync(options)
     end

data/lib/pg_easy_replicate/group.rb

@@ -20,6 +20,7 @@ module PgEasyReplicate
           column(:updated_at, Time, default: Sequel::CURRENT_TIMESTAMP)
           column(:started_at, Time)
           column(:failed_at, Time)
+          column(:recreate_indices_post_copy, TrueClass, default: true)
           column(:switchover_completed_at, Time)
         end
       ensure

data/lib/pg_easy_replicate/index_manager.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module PgEasyReplicate
+  module IndexManager
+    extend Helper
+
+    def self.drop_indices(
+      source_conn_string:,
+      target_conn_string:,
+      tables:,
+      schema:
+    )
+      logger.info("Dropping indices from target database")
+
+      fetch_indices(
+        conn_string: source_conn_string,
+        tables: tables,
+        schema: schema,
+      ).each do |index|
+        drop_sql = "DROP INDEX CONCURRENTLY #{index[:index_name]};"
+
+        Query.run(
+          query: drop_sql,
+          connection_url: target_conn_string,
+          schema: schema,
+          transaction: false,
+        )
+      end
+    end
+
+    def self.recreate_indices(
+      source_conn_string:,
+      target_conn_string:,
+      tables:,
+      schema:
+    )
+      logger.info("Recreating indices on target database")
+
+      indices =
+        fetch_indices(
+          conn_string: source_conn_string,
+          tables: tables,
+          schema: schema,
+        )
+      indices.each do |index|
+        create_sql =
+          "#{index[:index_definition].gsub("CREATE INDEX", "CREATE INDEX CONCURRENTLY IF NOT EXISTS")};"
+
+        Query.run(
+          query: create_sql,
+          connection_url: target_conn_string,
+          schema: schema,
+          transaction: false,
+        )
+      end
+    end
+
+    def self.fetch_indices(conn_string:, tables:, schema:)
+      return [] if tables.split(",").empty?
+      table_list = tables.split(",").map { |table| "'#{table}'" }.join(",")
+
+      sql = <<-SQL
+        SELECT
+            t.relname AS table_name,
+            i.relname AS index_name,
+            pg_get_indexdef(i.oid) AS index_definition
+        FROM
+            pg_class t,
+            pg_class i,
+            pg_index ix,
+            pg_namespace n
+        WHERE
+            t.oid = ix.indrelid
+            AND i.oid = ix.indexrelid
+            AND n.oid = t.relnamespace
+            AND t.relkind = 'r'  -- only find indexes of tables
+            AND ix.indisprimary = FALSE  -- exclude primary keys
+            AND n.nspname = '#{schema}'
+            AND t.relname IN (#{table_list})
+        ORDER BY
+            t.relname,
+            i.relname;
+      SQL
+      Query.run(query: sql, connection_url: conn_string, schema: schema)
+    end
+
+    def self.wait_for_replication_completion(group_name:)
+      loop do
+        break if Stats.all_tables_replicating?(group_name)
+        sleep(5)
+      end
+    end
+  end
+end

data/lib/pg_easy_replicate/orchestrate.rb

@@ -17,6 +17,15 @@ module PgEasyReplicate
             list: options[:tables],
           )
 
+        if options[:recreate_indices_post_copy]
+          IndexManager.drop_indices(
+            source_conn_string: source_db_url,
+            target_conn_string: target_db_url,
+            tables: tables,
+            schema: schema_name,
+          )
+        end
+
         create_publication(
           group_name: options[:group_name],
           conn_string: source_db_url,
@@ -40,6 +49,7 @@ module PgEasyReplicate
           table_names: tables,
           schema_name: schema_name,
           started_at: Time.now.utc,
+          recreate_indices_post_copy: options[:recreate_indices_post_copy],
         )
       rescue => e
         stop_sync(
@@ -221,7 +231,22 @@ module PgEasyReplicate
           tables: group[:table_names],
           schema: group[:schema_name],
         )
+
         watch_lag(group_name: group_name, lag: lag_delta_size || DEFAULT_LAG)
+
+        if group[:recreate_indices_post_copy]
+          IndexManager.wait_for_replication_completion(group_name: group_name)
+          IndexManager.recreate_indices(
+            source_conn_string: source_db_url,
+            target_conn_string: target_db_url,
+            tables: group[:table_names],
+            schema: group[:schema_name],
+          )
+        end
+
+        # Watch for lag again, because it could've grown during index recreation
+        watch_lag(group_name: group_name, lag: lag_delta_size || DEFAULT_LAG)
+
         revoke_connections_on_source_db(group_name)
         wait_for_remaining_catchup(group_name)
         refresh_sequences(

data/lib/pg_easy_replicate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEasyReplicate
-  VERSION = "0.1.12"
+  VERSION = "0.2.0"
 end

data/lib/pg_easy_replicate.rb

@@ -10,6 +10,7 @@ require "English"
 require "pg_easy_replicate/helper"
 require "pg_easy_replicate/version"
 require "pg_easy_replicate/query"
+require "pg_easy_replicate/index_manager"
 require "pg_easy_replicate/orchestrate"
 require "pg_easy_replicate/stats"
 require "pg_easy_replicate/group"
@@ -90,7 +91,7 @@ module PgEasyReplicate
       setup_internal_schema
 
       if options[:copy_schema]
-        logger.info("Setting up schema on targer database")
+        logger.info("Setting up schema on target database")
         copy_schema(
           source_conn_string: source_db_url,
           target_conn_string: target_db_url,

data/scripts/e2e-start.sh

@@ -13,6 +13,6 @@ export PGPASSWORD='james-bond123@7!'"'"''"'"'3aaR'
 # Bootstrap and cleanup
 echo "===== Performing Bootstrap and cleanup"
 bundle exec bin/pg_easy_replicate bootstrap -g cluster-1 --copy-schema
-bundle exec bin/pg_easy_replicate start_sync -g cluster-1 -s public
+bundle exec bin/pg_easy_replicate start_sync -g cluster-1 -s public --recreate-indices-post-copy
 bundle exec bin/pg_easy_replicate stats -g cluster-1
 bundle exec bin/pg_easy_replicate switchover -g cluster-1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_easy_replicate
 version: !ruby/object:Gem::Version
-  version: 0.1.12
+  version: 0.2.0
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-12-14 00:00:00.000000000 Z
+date: 2023-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai
@@ -62,16 +62,22 @@ dependencies:
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: prettier_print
   requirement: !ruby/object:Gem::Requirement
@@ -262,6 +268,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- assets/mascot.png
 - bin/pg_easy_replicate
 - bin/pg_easy_replicate_console
 - docker-compose.yml
@@ -269,6 +276,7 @@ files:
 - lib/pg_easy_replicate/cli.rb
 - lib/pg_easy_replicate/group.rb
 - lib/pg_easy_replicate/helper.rb
+- lib/pg_easy_replicate/index_manager.rb
 - lib/pg_easy_replicate/orchestrate.rb
 - lib/pg_easy_replicate/query.rb
 - lib/pg_easy_replicate/stats.rb
@@ -291,7 +299,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="