koyo-postgres-replication 0.1.3.pre → 0.1.5.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1cc8e3d452d94177ba7d15d098bbb38b949c45a4e25c22562b358f2858b8d541
-  data.tar.gz: f844b6eb897ab00c079be8e09a45398532657938ab9e2ae762ddd07c67f685be
+  metadata.gz: 8b45ea024487cd58842efb84ffb49a8a838a087c3302fc1411d02ec1afc646c2
+  data.tar.gz: 81900cffddb22af277e55be4c558316b9e0a61e3c9d9618f8d47ae1019f479ab
 SHA512:
-  metadata.gz: b7e17ddc565bcc1badd81dd6520506578e9eaf09be4598ace88b1336b8c8d797ea01e48724bad5217faf2442bcd04b5cce23a48f5feb048adc00d055e7c912d7
-  data.tar.gz: df0fd55f7909c6c401fbef598e618cd4837b22634400931b9942c50b650cde64b275c61ecdb7b0f82ab5f620a0b82b3851d9c4e40c20e9ecf7303b9e3628282c
+  metadata.gz: 52657842594b2e36b7ac7f658b2c3a56b3e4c6a4d7b22896dd1fade2daa34c7d7a4a9884c374930b055180b46078b72b2c6bdddf2a6b8a518ea67528e8b58b56
+  data.tar.gz: 7252cd8c30c701ee033cf7244ed668e9b23d9c9b0f48d958835eafe59a0419c0df60a05e09e580ebb798bec704e44708bd12905cf658669494ed58fa6a3082f6

data/.yardoc/checksums

@@ -3,12 +3,12 @@ lib/koyo/repl/log.rb 811d21aa6f8843589b66da79a96a4e8b755fcc2d
 lib/koyo/repl/mod.rb 92b54b56643b06fdc74adc65dc04460f631d4036
 lib/koyo/repl/data.rb cb847585c0f439b9f645b5ebd7fa3b96e08b94ff
 lib/koyo/repl/install.rb 9b19b7bb74d084882a1f726693bac2d693015daa
-lib/koyo/repl/railtie.rb acb36cbbe1b386ece7567990945adecb6b6e5b80
-lib/koyo/repl/version.rb 45af9c57eb86c840d6c0756f1cd0b8a2f4b00ac0
-lib/koyo/repl/data_row.rb 321caf0b080621f2d319fa152948cbe864c77294
-lib/koyo/repl/database.rb 39a304909ccf68d69d36c84e3036debd9f29f6ec
+lib/koyo/repl/railtie.rb 9c560c38f4663edda57aa39c9bb4bd9411daabbd
+lib/koyo/repl/version.rb d0d535b2aac27e3caa5eaeef69e117772e334f44
+lib/koyo/repl/data_row.rb 46bd0b440721f3b0ddcded3fef1817ca5d77cb35
+lib/koyo/repl/database.rb 719f2723b83a927e0524f6dea79393caaeba0514
 lib/koyo/repl/diagnostics.rb 93226c47642c5e20f615425aff8a0c0ba988b07b
 lib/koyo/repl/configuration.rb 8a32f819abf035340a3997d989ccafe0751d209e
-lib/koyo/repl/postgres_server.rb c8f557e1eec5b9be51ebd48002a3986eddbd6634
+lib/koyo/repl/postgres_server.rb 3d564dff9e7820962da02f8a383a607fb72110dc
 lib/koyo_postgres_replication.rb e3f4d905e3045b59f1f7c799d84cd9fde70eebd8
 lib/koyo/repl/event_handler_service.rb 99b67bdcf3e0e92a04cd01bf8d05a4569b45d218

data/Dockerfile

@@ -0,0 +1,29 @@
+FROM ruby:3.2.2-slim
+
+RUN apt-get update -qq && apt-get install -yq --no-install-recommends \
+    git \
+    build-essential \
+    gnupg2 \
+    less \
+    libpq-dev \
+    postgresql-client \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+ENV LANG=C.UTF-8 \
+  BUNDLE_JOBS=4 \
+  BUNDLE_RETRY=3
+  
+RUN gem update --system && gem install bundler && gem install rails
+
+WORKDIR /usr/src/app
+
+COPY Gemfile .
+COPY Gemfile.lock .
+
+COPY . .
+
+RUN bundle install
+
+COPY . .
+
+CMD ["irb"] 

data/Gemfile

@@ -6,6 +6,7 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 # Specify your gem's dependencies in koyo_repl.gemspec
 gemspec
 
+gem 'composite_primary_keys', '~> 14.0'
 gem 'pg', '~> 1.1'
 gem 'rails', '~> 7.0'
 gem 'rake', '~> 13.0'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    koyo-postgres-replication (0.1.2.pre)
+    koyo-postgres-replication (0.1.5.pre)
       pg (~> 1.1)
       rack (~> 2.0, >= 2.0.0)
       rails (~> 7.0)
@@ -78,6 +78,8 @@ GEM
     ast (2.4.2)
     base64 (0.1.1)
     builder (3.2.4)
+    composite_primary_keys (14.0.6)
+      activerecord (~> 7.0.2)
     concurrent-ruby (1.2.2)
     crass (1.0.6)
     date (3.3.3)
@@ -123,6 +125,8 @@ GEM
     net-smtp (0.3.3)
       net-protocol
     nio4r (2.5.9)
+    nokogiri (1.15.4-aarch64-linux)
+      racc (~> 1.4)
     nokogiri (1.15.4-arm64-darwin)
       racc (~> 1.4)
     parallel (1.23.0)
@@ -212,9 +216,11 @@ GEM
     zeitwerk (2.6.11)
 
 PLATFORMS
+  aarch64-linux
   arm64-darwin-22
 
 DEPENDENCIES
+  composite_primary_keys (~> 14.0)
   debug
   factory_bot_rails
   ffaker

data/README.md

@@ -1,6 +1,7 @@
 # Koyo::Postgres::Replication
 
 例 rei - Japanese for example 
+
 効用 koyo - Japanese for utility
 
 ## Replcation slots
@@ -170,6 +171,9 @@ for sql examples on how to interact with replication slots.
 
 ## Yard Doc
 
+Yard docs are up on
+[RubyDoc.info](https://rubydoc.info/github/wiseleyb/koyo-postgres-replication/main)
+
 Cheat sheets:
 * https://gist.github.com/chetan/1827484
 * https://kapeli.com/cheat_sheets/Yard.docset/Contents/Resources/Documents/index
@@ -188,6 +192,16 @@ TODO: update
 
 # Working with gem
 
+## Run specs
+
+```
+rspec spec
+```
+
+### Docker
+
+See [Docker wiki page](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Docker)
+
 ## Gem Build
 
 ```

data/changelog.md

@@ -1,5 +1,18 @@
 # Koyo Postgres Replication Changelog
 
+## 0.1.5.pre
+
+- Yard doc gemspec change to get it to show up in rubygems
+- Add Docker files
+
+## 0.1.4.pre
+
+- Add link to
+  [doc](https://rubydoc.info/github/wiseleyb/koyo-postgres-replication/main) in
+README
+- Fix issues around composite keys (multiple primary keys) [Issues
+  4](https://github.com/wiseleyb/koyo-postgres-replication/issues/4)
+
 ## 0.1.3.pre
 
 - republishing yanked gem

data/docker/postgres/Dockerfile

@@ -0,0 +1,3 @@
+FROM postgres:14
+RUN apt-get update -qq && apt-get install -yq --no-install-recommends \
+    postgresql-14-wal2json

data/docker-compose.yml

@@ -0,0 +1,34 @@
+version: '3.9'
+services:
+  db:
+    build: ./docker/postgres
+    image: postgres:13
+    ports:
+      - "5432:5432"
+    environment:
+      - POSTGRES_DB=rei_replication_test
+      - POSTGRES_PASSWORD=changeme
+      # - POSTGRES_USER=postgres
+    volumes:
+      - db_data:/var/lib/postgresql/data
+    command:
+      - "postgres"
+      - "-c"
+      - "wal_level=logical"
+  web:
+    build: .
+    image: rails-on-docker:1.3.0
+    stdin_open: true
+    tty: true
+    environment:
+      - DATABASE_URL=postgres://postgres:changeme@db
+      - BOOTSNAP_CACHE_DIR=/usr/local/bundle/_bootsnap
+      - HISTFILE=/usr/src/app/.dockerdev/.bash_history
+      - MALLOC_ARENA_MAX=2
+    volumes:
+      - .:/usr/src/app/:cached
+    depends_on:
+      - db
+volumes:
+  bundle:
+  db_data:

data/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+unset BUNDLE_PATH
+unset BUNDLE_BIN
+set -e
+exec "$@"

data/koyo-postgres-replication.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     'bug_tracker_uri' => "#{url}/issues",
     'changelog_uri' => "#{url}/blob/main/changelog.md",
-    'documentation' => 'https://www.rubydoc.info/github/wiseleyb/koyo-postgres-replication/main',
+    'documentation_uri' => 'https://www.rubydoc.info/github/wiseleyb/koyo-postgres-replication/main',
     'homepage_uri' => spec.homepage,
     'source_code_uri' => burl
   }

data/lib/koyo/repl/data_row.rb

@@ -11,8 +11,10 @@ module Koyo
                     :kind,          # insert/update/delete
                     :schema,        # always public for this - not needed
                     :table,         # table being changed
-                    :id,            # table.id
-                    :id_type,       # integer/uuid
+                    :id,            # table.id, null for composite keys
+                    :id_type,       # integer/uuid, null for composite keys
+                    :ids,           # for composite keys - array of ids
+                    :id_types,      # for composite keys - array of types
                     :columns,       # all columns from table - array
                     :column_types,  # all types of columns - array
                     :values         # all values from table - array
@@ -34,12 +36,12 @@ module Koyo
       # WARN: this breaks for multiple primary keys
       def check_set_primary_keys
         if @row['oldkeys']
-          if @row['oldkeys']['keynames'].size > 1
-            raise "This doesn't support multiple keys right now"
+          @ids = Array.new(@row['oldkeys']['keyvalues'])
+          @id_types = Array.new(@row['oldkeys']['keytypes'])
+          if @row['oldkeys']['keynames'].size == 1
+            @id = @ids.first
+            @id_type = @id_types.first
           end
-
-          @id = @row['oldkeys']['keyvalues'].first
-          @id_type = @row['oldkeys']['keytypes'].first
         else
           @id = val(:id)
           @id_type = type(:id)
@@ -49,13 +51,15 @@ module Koyo
       # Gets a value for a name from columnsvalues
       # @param name column name
       def val(name)
-        values[columns.index(name.to_s)]
+        idx = columns.index(name.to_s)
+        idx ? values[idx] : nil
       end
 
       # Get a val type from columntypes
       # @param name column name
       def type(name)
-        column_types[columns.index(name.to_s)]
+        idx = columns.index(name.to_s)
+        idx ? column_types[idx] : nil
       end
     end
   end

data/lib/koyo/repl/postgres_server.rb

@@ -62,6 +62,18 @@ module Koyo
         end
       end
 
+      # Does a single check of the replication slot
+      # If test_mode=true uses peek, which will
+      # leave data in the replication slot (for testing/debugging)
+      def check
+        read_sql_results.each do |sql_res|
+          rows = Koyo::Repl::Data.new(sql_res).rows # returns ReplDataRow
+          rows.each do |row|
+            check_row(row)
+          end
+        end
+      end
+
       # Attempts to re-establish DB connection. If it finds any other
       # error this is fatal and server crashes at this point
       # @param [StandardError] err Error that kicked off this retry loop
@@ -121,18 +133,6 @@ module Koyo
         @tick_tock = 0
       end
 
-      # Does a single check of the replication slot
-      # If test_mode=true uses peek, which will
-      # leave data in the replication slot (for testing/debugging)
-      def check
-        read_sql_results.each do |sql_res|
-          rows = Koyo::Repl::Data.new(sql_res).rows # returns ReplDataRow
-          rows.each do |row|
-            check_row(row)
-          end
-        end
-      end
-
       # Reads data from the replication slot
       # Handles test_mode (so will only peek if true)
       def read_sql_results

data/lib/koyo/repl/railtie.rb

@@ -31,6 +31,11 @@ module Koyo
               puts 'Running Koyo::Repl::PostgresServer.run!'
               Koyo::Repl::PostgresServer.run!
             end
+
+            desc 'Recreate replication slot'
+            task recreate: :environment do
+              Koyo::Repl::Database.drop_create_slot!
+            end
           end
         end
       end

data/lib/koyo/repl/version.rb

@@ -2,6 +2,6 @@
 
 module Koyo
   module Repl
-    VERSION = '0.1.3.pre'
+    VERSION = '0.1.5.pre'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: koyo-postgres-replication
 version: !ruby/object:Gem::Version
-  version: 0.1.3.pre
+  version: 0.1.5.pre
 platform: ruby
 authors:
 - Ben Wiseley
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-18 00:00:00.000000000 Z
+date: 2023-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -116,12 +116,16 @@ files:
 - ".yardoc/objects/root.dat"
 - ".yardoc/proxy_types"
 - ".yardopts"
+- Dockerfile
 - Gemfile
 - Gemfile.lock
 - MIT-LICENSE
 - README.md
 - Rakefile
 - changelog.md
+- docker-compose.yml
+- docker/postgres/Dockerfile
+- entrypoint.sh
 - koyo-postgres-replication.gemspec
 - lib/koyo.rb
 - lib/koyo/repl/configuration.rb
@@ -146,7 +150,7 @@ licenses:
 metadata:
   bug_tracker_uri: https://github.com/wiseleyb/koyo-postgres-replication/issues
   changelog_uri: https://github.com/wiseleyb/koyo-postgres-replication/blob/main/changelog.md
-  documentation: https://www.rubydoc.info/github/wiseleyb/koyo-postgres-replication/main
+  documentation_uri: https://www.rubydoc.info/github/wiseleyb/koyo-postgres-replication/main
   homepage_uri: https://github.com/wiseleyb
   source_code_uri: https://github.com/wiseleyb
 post_install_message: