koyo-postgres-replication 0.1.0.pre

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 78e654e0e390dedac52210540c5dafdb1c189b1bbb01968c8bd09b5bf5ee250f
+  data.tar.gz: 9dfdec20a290ae9d30ac91f7cfb404530053f883c9f12a57f678dd3665f4be8a
+SHA512:
+  metadata.gz: 97987ad66fbed7a4c14124de1776e771c3e54fb1c70b8f7fca6f588fc054dd858af38b753d2578e65aaefc25e9c048d86af49d6d3f6822ccbdadc7a76d42fbb9
+  data.tar.gz: d3fd6f95478bbf33830f7f8c85fcc3b99d9599987fea81cdd639f997d625bfd8fdfc99e9b4881da9ee6f703665d2019f9824ffc5ce38474828918c457abf82c6

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,23 @@
+Gemspec/RequiredRubyVersion:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Layout/LineLength:
+  Max: 90
+
+AllCops:
+  DisplayCopNames: true
+  Exclude:
+    - 'spec/dummy/config/**/*'
+    - 'spec/dummy/db/**/*'

data/.yardoc/checksums

@@ -0,0 +1,14 @@
+lib/koyo.rb 0baca6fa8b0186eea9fee1eff1bba6853aa2e691
+lib/koyo/repl/log.rb 811d21aa6f8843589b66da79a96a4e8b755fcc2d
+lib/koyo/repl/mod.rb 92b54b56643b06fdc74adc65dc04460f631d4036
+lib/koyo/repl/data.rb cb847585c0f439b9f645b5ebd7fa3b96e08b94ff
+lib/koyo/repl/install.rb 9b19b7bb74d084882a1f726693bac2d693015daa
+lib/koyo/repl/railtie.rb acb36cbbe1b386ece7567990945adecb6b6e5b80
+lib/koyo/repl/version.rb 94239a7a9719294e13a38b878da1d328b44fb3f5
+lib/koyo/repl/data_row.rb 321caf0b080621f2d319fa152948cbe864c77294
+lib/koyo/repl/database.rb 39a304909ccf68d69d36c84e3036debd9f29f6ec
+lib/koyo/repl/diagnostics.rb 03b91c5ea3dbe1e337f4427d8f3261ebedb45a70
+lib/koyo/repl/configuration.rb 17858828602d696a584a9e3c239c0a5d94faa738
+lib/koyo/repl/postgres_server.rb 2569fa642e55f6032e6ea73e9ab8762018d3cd6c
+lib/koyo_postgres_replication.rb e3f4d905e3045b59f1f7c799d84cd9fde70eebd8
+lib/koyo/repl/event_handler_service.rb 99b67bdcf3e0e92a04cd01bf8d05a4569b45d218

data/Gemfile

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+# Specify your gem's dependencies in koyo_repl.gemspec
+gemspec
+
+gem 'rails', '~> 7.0'
+gem 'rake', '~> 13.0'
+gem 'pg', '~> 1.1'
+
+group :development, :test do
+  gem 'debug', platforms: %i[mri mingw x64_mingw]
+  gem 'factory_bot_rails'
+  gem 'ffaker'
+  gem 'rspec-rails', '~> 6.0'
+  gem 'rubocop'
+  gem 'yard'
+end

data/Gemfile.lock

@@ -0,0 +1,230 @@
+PATH
+  remote: .
+  specs:
+    koyo-postgres-replication (0.1.0.pre)
+      pg (~> 1.1)
+      rack (~> 2.0, >= 2.0.0)
+      rails (~> 7.0)
+      rake (~> 13.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actioncable (7.0.7)
+      actionpack (= 7.0.7)
+      activesupport (= 7.0.7)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+    actionmailbox (7.0.7)
+      actionpack (= 7.0.7)
+      activejob (= 7.0.7)
+      activerecord (= 7.0.7)
+      activestorage (= 7.0.7)
+      activesupport (= 7.0.7)
+      mail (>= 2.7.1)
+      net-imap
+      net-pop
+      net-smtp
+    actionmailer (7.0.7)
+      actionpack (= 7.0.7)
+      actionview (= 7.0.7)
+      activejob (= 7.0.7)
+      activesupport (= 7.0.7)
+      mail (~> 2.5, >= 2.5.4)
+      net-imap
+      net-pop
+      net-smtp
+      rails-dom-testing (~> 2.0)
+    actionpack (7.0.7)
+      actionview (= 7.0.7)
+      activesupport (= 7.0.7)
+      rack (~> 2.0, >= 2.2.4)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actiontext (7.0.7)
+      actionpack (= 7.0.7)
+      activerecord (= 7.0.7)
+      activestorage (= 7.0.7)
+      activesupport (= 7.0.7)
+      globalid (>= 0.6.0)
+      nokogiri (>= 1.8.5)
+    actionview (7.0.7)
+      activesupport (= 7.0.7)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activejob (7.0.7)
+      activesupport (= 7.0.7)
+      globalid (>= 0.3.6)
+    activemodel (7.0.7)
+      activesupport (= 7.0.7)
+    activerecord (7.0.7)
+      activemodel (= 7.0.7)
+      activesupport (= 7.0.7)
+    activestorage (7.0.7)
+      actionpack (= 7.0.7)
+      activejob (= 7.0.7)
+      activerecord (= 7.0.7)
+      activesupport (= 7.0.7)
+      marcel (~> 1.0)
+      mini_mime (>= 1.1.0)
+    activesupport (7.0.7)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    ast (2.4.2)
+    base64 (0.1.1)
+    builder (3.2.4)
+    concurrent-ruby (1.2.2)
+    crass (1.0.6)
+    date (3.3.3)
+    debug (1.8.0)
+      irb (>= 1.5.0)
+      reline (>= 0.3.1)
+    diff-lcs (1.5.0)
+    erubi (1.12.0)
+    factory_bot (6.2.1)
+      activesupport (>= 5.0.0)
+    factory_bot_rails (6.2.0)
+      factory_bot (~> 6.2.0)
+      railties (>= 5.0.0)
+    ffaker (2.21.0)
+    globalid (1.1.0)
+      activesupport (>= 5.0)
+    i18n (1.14.1)
+      concurrent-ruby (~> 1.0)
+    io-console (0.6.0)
+    irb (1.7.4)
+      reline (>= 0.3.6)
+    json (2.6.3)
+    language_server-protocol (3.17.0.3)
+    loofah (2.21.3)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.12.0)
+    mail (2.8.1)
+      mini_mime (>= 0.1.1)
+      net-imap
+      net-pop
+      net-smtp
+    marcel (1.0.2)
+    method_source (1.0.0)
+    mini_mime (1.1.5)
+    minitest (5.19.0)
+    net-imap (0.3.7)
+      date
+      net-protocol
+    net-pop (0.1.2)
+      net-protocol
+    net-protocol (0.2.1)
+      timeout
+    net-smtp (0.3.3)
+      net-protocol
+    nio4r (2.5.9)
+    nokogiri (1.15.4-arm64-darwin)
+      racc (~> 1.4)
+    parallel (1.23.0)
+    parser (3.2.2.3)
+      ast (~> 2.4.1)
+      racc
+    pg (1.5.3)
+    racc (1.7.1)
+    rack (2.2.8)
+    rack-test (2.1.0)
+      rack (>= 1.3)
+    rails (7.0.7)
+      actioncable (= 7.0.7)
+      actionmailbox (= 7.0.7)
+      actionmailer (= 7.0.7)
+      actionpack (= 7.0.7)
+      actiontext (= 7.0.7)
+      actionview (= 7.0.7)
+      activejob (= 7.0.7)
+      activemodel (= 7.0.7)
+      activerecord (= 7.0.7)
+      activestorage (= 7.0.7)
+      activesupport (= 7.0.7)
+      bundler (>= 1.15.0)
+      railties (= 7.0.7)
+    rails-dom-testing (2.2.0)
+      activesupport (>= 5.0.0)
+      minitest
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.6.0)
+      loofah (~> 2.21)
+      nokogiri (~> 1.14)
+    railties (7.0.7)
+      actionpack (= 7.0.7)
+      activesupport (= 7.0.7)
+      method_source
+      rake (>= 12.2)
+      thor (~> 1.0)
+      zeitwerk (~> 2.5)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.8.1)
+    reline (0.3.7)
+      io-console (~> 0.5)
+    rexml (3.2.6)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.6)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-rails (6.0.3)
+      actionpack (>= 6.1)
+      activesupport (>= 6.1)
+      railties (>= 6.1)
+      rspec-core (~> 3.12)
+      rspec-expectations (~> 3.12)
+      rspec-mocks (~> 3.12)
+      rspec-support (~> 3.12)
+    rspec-support (3.12.1)
+    rubocop (1.56.0)
+      base64 (~> 0.1.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.2.2.3)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.29.0)
+      parser (>= 3.2.1.0)
+    ruby-progressbar (1.13.0)
+    thor (1.2.2)
+    timeout (0.4.0)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.4.2)
+    websocket-driver (0.7.6)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
+    yard (0.9.34)
+    zeitwerk (2.6.11)
+
+PLATFORMS
+  arm64-darwin-22
+
+DEPENDENCIES
+  debug
+  factory_bot_rails
+  ffaker
+  koyo-postgres-replication!
+  pg (~> 1.1)
+  rails (~> 7.0)
+  rake (~> 13.0)
+  rspec-rails (~> 6.0)
+  rubocop
+  yard
+
+BUNDLED WITH
+   2.4.18

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2023 Ben Wiseley
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,192 @@
+# Koyo::Postgres::Replication
+
+## Replcation slots
+
+This gem tries to simplify dealing with a `replication slot` in Postgres. 
+
+### What is a replication slot?
+
+Please see [the wiki page that discusses replication
+slots](https://github.com/wiseleyb/koyo-postgres-replication/wiki/What-is-replication%3F)
+
+### Why would you use this?
+
+Example: You have a no-sql store (like Elastic-Search) that needs to be in-sync
+with your database. You could do this "the Rails way" with `after-save` type
+patterns. You could do this with a service type architecture that updates
+things. But if you have non-rails teams updating data, or ever update the
+database via SQL, this becomes more complex, duplicates delicate/error-prone
+work between teams, or just isn't possible. Going the microservice approach
+also isn't ideal for performance reasons (you never want to updating a really
+busy DB via API). 
+
+Please see [the
+wiki](https://github.com/wiseleyb/koyo-postgres-replication/wiki/What-is-replication%3F)
+for more discussion on this.
+
+## Quick Start
+
+You need to configure Postgres for this first. This isn't enabled by default.
+See [Configuring Postgres for Replication in the
+wiki](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Configuring-Postgres-for-Replication)
+
+Add to Gemfile:
+
+```
+gem 'koyo-postgres-replication', 
+    git: 'https://github.com/wiseleyb/koyo-postgres-replication',
+    require: 'koyo'
+```
+
+Then:
+
+```
+bundle install
+bundle exec rake koyo:repl:install
+```
+
+This adds files:
+
+* `config/initializers/koyo_postgres_replication_config.rb`: which is commented
+  on various ways to override various settings.
+* `app/models/koyo_repl_handler_service.rb`: which has call backs for every
+  replication event, log events, and errors. This file is also heavily
+commented.
+* `app/models/koyo_repl_model_example.rb`: is an example of how to add
+  replication monitoring on a model level. You can delete this file - it's just
+there for a simple example. 
+
+### Run Diagnostics
+
+You need to create a replication slot for each database. This requires admin
+access. In
+[config](https://github.com/wiseleyb/koyo-postgres-replication/blob/main/lib/koyo/repl/templates/koyo_postgres_replication_config.txt)
+you can setup a separate `config/database.yml` connection to limit admin level
+postgres access if your company prefers that.
+
+```
+# runs basic diagnostics - look for Error in this list and fix any issues
+bundle exec rake koyo:repl:diagnostics
+```
+
+Which will output something like:
+
+```
+--------------------------------------------------------------------------------
+Koyo::Repl::Diagnostic
+source=KoyoReplication logid=f3a8f68d3e level=info message=Init: Finding models that support koyo_repl_handler
+source=KoyoReplication logid=410b6fd0d0 level=info message=Init: ignoring model SchemaMigration
+source=KoyoReplication logid=b429e47251 level=info message=Init: ignoring model ArInternalMetadatum
+source=KoyoReplication logid=6532a8ec76 level=info message=Init: registering handler ["users", "User"]
+Config settings:
+  auto_create_replication_slot: true
+  config_prefix: KOYO_REPL
+  db_conn:
+  slot: koyo_repl_rei_postgres_replication_development_development
+  sql_delay: 1
+  test_mode: false
+Replication slot exists: true
+Registered tables:
+  users: User
+Can connect to db: true
+Connection adapter: PostgreSQL
+Wal Level (should be 'logical'): logical
+Can access replication slot: true
+Replication slot count: 0
+--------------------------------------------------------------------------------
+```
+
+If there are errors you'll need to fix those first before running the server.
+
+### Run the server
+
+```
+bundle exec rake koyo:repl:run_server
+```
+
+Now - when you create/update/delete data in the configured database you should
+be getting callbacks in
+`app/models/koyo_repl_handler_service.rb#koyo_handle_all_replication(row)` and
+in any model `app/models/{some-model}#handle_replication(row)` that implements
+callbacks.
+
+# Row data in callbacks
+
+See [wiki page on DataRow for raw data examples of what you get from
+Postgres](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Koyo::Repl::DataRow-data-spec)
+
+# Processing callbacks
+
+Both `koyo_repl_handler_service#koyo_handle_all_replication(row)` and
+`{some-model}#handle_replication(row)` need to be REALLY fast. You shouldn't do
+database updates from this code (or risk infinite loops) and, if you're doing
+something like updating an API you should async that via Sidekiq, ActiveJob,
+etc. Keep in mind that, if you're running multiple Sidekiq servers that things
+might not be processed in the same order. Please see the [wiki page on
+processing
+callbacks](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Processing-callbacks)
+for more on this.
+
+# Testing
+
+The following assumes you're using rspec (open to PRs supporting other test
+frameworks though - I just don't use those)
+
+By default (and definitely the fastest way) specs run inside
+transactions that are rolled back after each spec runs. You need to use
+[Database Cleaner](https://github.com/DatabaseCleaner/database_cleaner)
+truncation approach to test this stuff. Please see the wiki on
+[testing](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Testing)
+for what's needed if you want to run replication slot tests.
+
+# Sample apps
+
+These are simple Rails version-specific demo apps with Docker files
+
+* [Rails 7 Example](https://github.com/wiseleyb/rei-postgres-replication/tree/rails-7-example)
+* Rails 6 Example: coming soon
+* Rails 5 Example: coming soon
+* Rails 4 Example: coming soon
+* Rails 3 Example: probably won't do
+ 
+# Technical 
+
+See wiki page on [creating this
+gem](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Creating-this-GEM)
+if you're interested how to do gems like these.
+
+## Working with the gem
+
+See wiki page on [working with this
+gem](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Developing-on-this-GEM)
+for basics of debugging and working with.
+
+## Working with replication slots
+
+See
+[koyo::repl::database](https://github.com/wiseleyb/koyo-postgres-replication/blob/main/lib/koyo/repl/database.rb)
+for sql examples on how to interact with replication slots.
+
+## Yard Doc
+
+Cheat sheets:
+* https://gist.github.com/chetan/1827484
+* https://kapeli.com/cheat_sheets/Yard.docset/Contents/Resources/Documents/index
+
+Build yard docs: `yard`
+View docs: `yard server` then open https://localhost:8808
+
+# Contributing
+
+TODO: update
+
+* follow github guide
+* run/add specs
+* run rubocop
+* add/run yard
+
+# TODO
+
+* add monitoring helpers
+* add support for multiple primary keys
+* add badges like in https://raw.githubusercontent.com/rubocop/rubocop/master/README.md

data/Rakefile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+APP_RAKEFILE = File.expand_path('spec/dummy/Rakefile', __dir__)
+load 'rails/tasks/engine.rake'
+
+load 'rails/tasks/statistics.rake'
+
+require 'bundler/gem_tasks'

data/changelog.md

@@ -0,0 +1,14 @@
+# Koyo Postgres Replication Changelog
+
+## 0.1.0.pre
+
+- rails 7 plugin
+- postgres replication monitor
+- install script 
+- documentation
+- basic specs
+- example rails project
+- diagnostic tools
+- utility class for working with replication slots
+- functional - but not tested in real life yet
+

data/koyo-postgres-replication.gemspec

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative 'lib/koyo'
+
+burl = 'https://github.com/wiseleyb'
+url = "#{burl}/koyo-postgres-replication"
+
+Gem::Specification.new do |spec|
+  spec.name        = 'koyo-postgres-replication'
+  spec.version     = Koyo::Repl::VERSION
+  spec.authors     = ['Ben Wiseley']
+  spec.email       = ['wiseleyb@gmail.com']
+  spec.homepage    = burl
+  spec.summary     = 'Postgres Replication'
+  spec.description = 'Simple Postgres replication helper'
+  spec.license     = 'MIT'
+
+  spec.required_ruby_version = '>= 2.7.0'
+
+  # spec.metadata["allowed_push_host"] = ''
+  spec.metadata['changelog_uri'] = "#{url}/changelog.md"
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = url
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) ==
+       __FILE__) ||
+        f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor])
+    end
+  end
+  spec.bindir = 'bin'
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'pg', '~> 1.1'
+  spec.add_dependency 'rack', '~> 2.0', '>= 2.0.0'
+  spec.add_dependency 'rails', '~> 7.0'
+  spec.add_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rspec-rails', '~> 3.1'
+  spec.add_development_dependency 'yard', '~> 0.9'
+
+  spec.has_rdoc = 'yard'
+end