koyo-postgres-replication 0.1.1.pre → 0.1.4.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52f66b5b4c592753e8cb81e6291a035289cce3f58bc610e4d78de5a1c0bb6fa9
-  data.tar.gz: bbb086947453e1937f052ed86e6dd68efd4797b5a2d2ecaaccb77de92185832a
+  metadata.gz: d7e309ca1f76d89a85e0d84b66fd54b629e7060ffd089a6a4383099230ccad9f
+  data.tar.gz: d387c7c45a91cf7500c83188688024fd8c08c97c4298fcc015d1d7b12adc425b
 SHA512:
-  metadata.gz: cd86fc705158b786bb847522d25bcccd881c55758a5f0857d8696fa6cee60972c1ca8ff0bf2bced4ad6444c5891a32acc19ed586d738eaea0b8c3cd34dac2218
-  data.tar.gz: 7c48ed00f79b044873fafbef2e9ced4828caa50d0804d2720d220c49052fe3db67a22230312db1de91cc4ddaad2d21f5cdda04dfaf6efcf13bcbca176184c737
+  metadata.gz: 69dae55f084fc5d4206164d47e312d5bcbf530397190563126097fc3e0250760537ae1930b68228e18f7490e929e374af67663bb3ee8b6da99460dc718951bc6
+  data.tar.gz: 4c53b1c6a422c78f70885ac5db3a4951b8323b8ba9707771915cd4cbe933274d2697c467c2b75586ca9c77e2e5fd299cf8bdfdc5b011a39dc6efa2ab6288bb82

data/.rubocop.yml

@@ -17,6 +17,7 @@ Layout/LineLength:
   Max: 90
 
 AllCops:
+  TargetRubyVersion: 3.2.2
   DisplayCopNames: true
   Exclude:
     - 'spec/dummy/config/**/*'

data/.yardoc/checksums

@@ -3,12 +3,12 @@ lib/koyo/repl/log.rb 811d21aa6f8843589b66da79a96a4e8b755fcc2d
 lib/koyo/repl/mod.rb 92b54b56643b06fdc74adc65dc04460f631d4036
 lib/koyo/repl/data.rb cb847585c0f439b9f645b5ebd7fa3b96e08b94ff
 lib/koyo/repl/install.rb 9b19b7bb74d084882a1f726693bac2d693015daa
-lib/koyo/repl/railtie.rb acb36cbbe1b386ece7567990945adecb6b6e5b80
-lib/koyo/repl/version.rb 94239a7a9719294e13a38b878da1d328b44fb3f5
-lib/koyo/repl/data_row.rb 321caf0b080621f2d319fa152948cbe864c77294
-lib/koyo/repl/database.rb 39a304909ccf68d69d36c84e3036debd9f29f6ec
-lib/koyo/repl/diagnostics.rb 03b91c5ea3dbe1e337f4427d8f3261ebedb45a70
-lib/koyo/repl/configuration.rb 17858828602d696a584a9e3c239c0a5d94faa738
-lib/koyo/repl/postgres_server.rb 2569fa642e55f6032e6ea73e9ab8762018d3cd6c
+lib/koyo/repl/railtie.rb 9c560c38f4663edda57aa39c9bb4bd9411daabbd
+lib/koyo/repl/version.rb d0d535b2aac27e3caa5eaeef69e117772e334f44
+lib/koyo/repl/data_row.rb 46bd0b440721f3b0ddcded3fef1817ca5d77cb35
+lib/koyo/repl/database.rb 719f2723b83a927e0524f6dea79393caaeba0514
+lib/koyo/repl/diagnostics.rb 93226c47642c5e20f615425aff8a0c0ba988b07b
+lib/koyo/repl/configuration.rb 8a32f819abf035340a3997d989ccafe0751d209e
+lib/koyo/repl/postgres_server.rb 3d564dff9e7820962da02f8a383a607fb72110dc
 lib/koyo_postgres_replication.rb e3f4d905e3045b59f1f7c799d84cd9fde70eebd8
 lib/koyo/repl/event_handler_service.rb 99b67bdcf3e0e92a04cd01bf8d05a4569b45d218

data/.yardopts

@@ -0,0 +1,7 @@
+--protected
+--no-private
+--embed-mixin ClassMethods
+-
+README.md
+CHANGELOG.md
+MIT-LICENSE

data/Gemfile

@@ -6,6 +6,7 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 # Specify your gem's dependencies in koyo_repl.gemspec
 gemspec
 
+gem 'composite_primary_keys', '~> 14.0'
 gem 'pg', '~> 1.1'
 gem 'rails', '~> 7.0'
 gem 'rake', '~> 13.0'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    koyo-postgres-replication (0.1.1.pre)
+    koyo-postgres-replication (0.1.4.pre)
       pg (~> 1.1)
       rack (~> 2.0, >= 2.0.0)
       rails (~> 7.0)
@@ -78,6 +78,8 @@ GEM
     ast (2.4.2)
     base64 (0.1.1)
     builder (3.2.4)
+    composite_primary_keys (14.0.6)
+      activerecord (~> 7.0.2)
     concurrent-ruby (1.2.2)
     crass (1.0.6)
     date (3.3.3)
@@ -215,6 +217,7 @@ PLATFORMS
   arm64-darwin-22
 
 DEPENDENCIES
+  composite_primary_keys (~> 14.0)
   debug
   factory_bot_rails
   ffaker

data/README.md

@@ -1,6 +1,7 @@
 # Koyo::Postgres::Replication
 
 例 rei - Japanese for example 
+
 効用 koyo - Japanese for utility
 
 ## Replcation slots
@@ -36,9 +37,7 @@ wiki](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Configuring-Pos
 Add to Gemfile:
 
 ```
-gem 'koyo-postgres-replication', 
-    git: 'https://github.com/wiseleyb/koyo-postgres-replication',
-    require: 'koyo'
+gem 'koyo-postgres-replication', require: 'koyo'
 ```
 
 Then:
@@ -172,6 +171,9 @@ for sql examples on how to interact with replication slots.
 
 ## Yard Doc
 
+Yard docs are up on
+[RubyDoc.info](https://rubydoc.info/github/wiseleyb/koyo-postgres-replication/main)
+
 Cheat sheets:
 * https://gist.github.com/chetan/1827484
 * https://kapeli.com/cheat_sheets/Yard.docset/Contents/Resources/Documents/index

data/changelog.md

@@ -1,5 +1,31 @@
 # Koyo Postgres Replication Changelog
 
+## 0.1.4.pre
+
+- Add link to
+  [doc](https://rubydoc.info/github/wiseleyb/koyo-postgres-replication/main) in
+README
+- Fix issues around composite keys (multiple primary keys) [Issues
+  4](https://github.com/wiseleyb/koyo-postgres-replication/issues/4)
+
+## 0.1.3.pre
+
+- republishing yanked gem
+
+## 0.1.2.pre
+
+- bump TargetRubyVersion for Rubocop to 3.2.2
+- fix [Issue
+  2](https://github.com/wiseleyb/koyo-postgres-replication/issues/2) - gemspec
+config around Yard doc
+- Enter retry loop instead of crashing when db connection goes away
+
+## 0.1.1.pre
+
+- remove deprecated has_rdoc from gem spec
+- fix [Issue 1](https://github.com/wiseleyb/koyo-postgres-replication/issues/1)
+  - remove recursive run, replace with simple loop
+
 ## 0.1.0.pre
 
 - rails 7 plugin
@@ -12,7 +38,3 @@
 - utility class for working with replication slots
 - functional - but not tested in real life yet
 
-## 0.1.1.pre
-
-- remove deprecated has_rdoc from gem spec
-- fix [Issue 1](https://github.com/wiseleyb/koyo-postgres-replication/issues/1) - remove recursive run, replace with simple loop

data/koyo-postgres-replication.gemspec

@@ -17,10 +17,13 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.7.0'
 
-  # spec.metadata["allowed_push_host"] = ''
-  spec.metadata['changelog_uri'] = "#{url}/changelog.md"
-  spec.metadata['homepage_uri'] = spec.homepage
-  spec.metadata['source_code_uri'] = burl
+  spec.metadata = {
+    'bug_tracker_uri' => "#{url}/issues",
+    'changelog_uri' => "#{url}/blob/main/changelog.md",
+    'documentation' => 'https://www.rubydoc.info/github/wiseleyb/koyo-postgres-replication/main',
+    'homepage_uri' => spec.homepage,
+    'source_code_uri' => burl
+  }
 
   spec.files = Dir.chdir(__dir__) do
     `git ls-files -z`.split("\x0").reject do |f|

data/lib/koyo/repl/data_row.rb

@@ -11,8 +11,10 @@ module Koyo
                     :kind,          # insert/update/delete
                     :schema,        # always public for this - not needed
                     :table,         # table being changed
-                    :id,            # table.id
-                    :id_type,       # integer/uuid
+                    :id,            # table.id, null for composite keys
+                    :id_type,       # integer/uuid, null for composite keys
+                    :ids,           # for composite keys - array of ids
+                    :id_types,      # for composite keys - array of types
                     :columns,       # all columns from table - array
                     :column_types,  # all types of columns - array
                     :values         # all values from table - array
@@ -34,12 +36,12 @@ module Koyo
       # WARN: this breaks for multiple primary keys
       def check_set_primary_keys
         if @row['oldkeys']
-          if @row['oldkeys']['keynames'].size > 1
-            raise "This doesn't support multiple keys right now"
+          @ids = Array.new(@row['oldkeys']['keyvalues'])
+          @id_types = Array.new(@row['oldkeys']['keytypes'])
+          if @row['oldkeys']['keynames'].size == 1
+            @id = @ids.first
+            @id_type = @id_types.first
           end
-
-          @id = @row['oldkeys']['keyvalues'].first
-          @id_type = @row['oldkeys']['keytypes'].first
         else
           @id = val(:id)
           @id_type = type(:id)
@@ -49,13 +51,15 @@ module Koyo
       # Gets a value for a name from columnsvalues
       # @param name column name
       def val(name)
-        values[columns.index(name.to_s)]
+        idx = columns.index(name.to_s)
+        idx ? values[idx] : nil
       end
 
       # Get a val type from columntypes
       # @param name column name
       def type(name)
-        column_types[columns.index(name.to_s)]
+        idx = columns.index(name.to_s)
+        idx ? column_types[idx] : nil
       end
     end
   end

data/lib/koyo/repl/database.rb

@@ -19,6 +19,7 @@ module Koyo
           conn_name = Koyo::Repl.config.database_name
 
           unless conn_name
+            ActiveRecord::Base.establish_connection
             @conn = ActiveRecord::Base.connection
             return @conn
           end
@@ -35,6 +36,14 @@ module Koyo
           @conn
         end
 
+        # Attempts to re-establish db connection
+        # This would be used when something happens like the postgres
+        # server restarts
+        def re_establish_conn
+          @conn = nil
+          conn
+        end
+
         # Reads from the replication slot.
         # Reading from this marks the rows read (so you won't see them again)
         # For testing you can use `peek_slot` if you want to - which will keep

data/lib/koyo/repl/postgres_server.rb

@@ -51,24 +51,69 @@ module Koyo
             tick_tock
           # Possibly fatal errors
           rescue ActiveRecord::StatementInvalid => e
-            if e.cause.exception.is_a?(PG::ConnectionBad)
-              Koyo::Repl::EventHandlerService.koyo_error(e)
-              msg = "SHUTTING DOWN. Fatal Error in ReplPostgresServer: #{e.message}"
-              log_repl_fatal(msg, err: e)
-              break
-            else
-              log_recoverable_error(e)
-            end
-          # recoverable error
+            Koyo::Repl::EventHandlerService.koyo_error(e)
+            @retry = 0
+            break unless attempt_reconnect(e)
           rescue StandardError => e
             log_recoverable_error(e)
           end
 
-          success = system("sleep #{Koyo::Repl.config.sql_delay}")
-          break unless success
+          break unless sleep_success?
         end
       end
 
+      # Does a single check of the replication slot
+      # If test_mode=true uses peek, which will
+      # leave data in the replication slot (for testing/debugging)
+      def check
+        read_sql_results.each do |sql_res|
+          rows = Koyo::Repl::Data.new(sql_res).rows # returns ReplDataRow
+          rows.each do |row|
+            check_row(row)
+          end
+        end
+      end
+
+      # Attempts to re-establish DB connection. If it finds any other
+      # error this is fatal and server crashes at this point
+      # @param [StandardError] err Error that kicked off this retry loop
+      def attempt_reconnect(err)
+        @retry ||= 0
+        msg = "Error: Attempting to reconnect to DB. Retry: #{@retry}."
+        log_recoverable_error(err)
+
+        return false unless sleep_success?
+
+        Koyo::Repl::Database.re_establish_conn
+        msg = 'Re-established DB connection'
+        log_repl_info(msg)
+        true
+      rescue StandardError => e
+        unless bad_connection_error?(e)
+          msg = 'Fatal: Found erorr unrelated to PG::ConnectionBad while '\
+                "trying to reconnect: Error: #{e.message}"
+          log_repl_fatal(msg, err: e)
+          return false
+        end
+
+        @retry += 1
+        retry
+      end
+
+      # Test error to see if it's db:connection related
+      # Mostly here for specs
+      # @param [StandardError] err Error to check
+      def bad_connection_error?(err)
+        err.cause.exception.is_a?(PG::ConnectionBad)
+      end
+
+      # Did sleep succeed... this fails when you do something like hit
+      # ctrl-c on the server
+      # Mostly here for specs
+      def sleep_success?
+        system("sleep #{Koyo::Repl.config.sql_delay}")
+      end
+
       # Handles erros that aren't fatal. Calls back to
       # Koyo::Repl::Log@log_repl_error which calls back
       # to KoyoReplHandlerServer@log_repl_error
@@ -88,18 +133,6 @@ module Koyo
         @tick_tock = 0
       end
 
-      # Does a single check of the replication slot
-      # If test_mode=true uses peek, which will
-      # leave data in the replication slot (for testing/debugging)
-      def check
-        read_sql_results.each do |sql_res|
-          rows = Koyo::Repl::Data.new(sql_res).rows # returns ReplDataRow
-          rows.each do |row|
-            check_row(row)
-          end
-        end
-      end
-
       # Reads data from the replication slot
       # Handles test_mode (so will only peek if true)
       def read_sql_results

data/lib/koyo/repl/railtie.rb

@@ -31,6 +31,11 @@ module Koyo
               puts 'Running Koyo::Repl::PostgresServer.run!'
               Koyo::Repl::PostgresServer.run!
             end
+
+            desc 'Recreate replication slot'
+            task recreate: :environment do
+              Koyo::Repl::Database.drop_create_slot!
+            end
           end
         end
       end

data/lib/koyo/repl/version.rb

@@ -2,6 +2,6 @@
 
 module Koyo
   module Repl
-    VERSION = '0.1.1.pre'
+    VERSION = '0.1.4.pre'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: koyo-postgres-replication
 version: !ruby/object:Gem::Version
-  version: 0.1.1.pre
+  version: 0.1.4.pre
 platform: ruby
 authors:
 - Ben Wiseley
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-18 00:00:00.000000000 Z
+date: 2023-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -115,6 +115,7 @@ files:
 - ".yardoc/object_types"
 - ".yardoc/objects/root.dat"
 - ".yardoc/proxy_types"
+- ".yardopts"
 - Gemfile
 - Gemfile.lock
 - MIT-LICENSE
@@ -143,7 +144,9 @@ homepage: https://github.com/wiseleyb
 licenses:
 - MIT
 metadata:
-  changelog_uri: https://github.com/wiseleyb/koyo-postgres-replication/changelog.md
+  bug_tracker_uri: https://github.com/wiseleyb/koyo-postgres-replication/issues
+  changelog_uri: https://github.com/wiseleyb/koyo-postgres-replication/blob/main/changelog.md
+  documentation: https://www.rubydoc.info/github/wiseleyb/koyo-postgres-replication/main
   homepage_uri: https://github.com/wiseleyb
   source_code_uri: https://github.com/wiseleyb
 post_install_message: