koyo-postgres-replication 0.1.0.pre
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.DS_Store +0 -0
- data/.rspec +1 -0
- data/.rubocop.yml +23 -0
- data/.yardoc/checksums +14 -0
- data/.yardoc/complete +0 -0
- data/.yardoc/object_types +0 -0
- data/.yardoc/objects/root.dat +0 -0
- data/.yardoc/proxy_types +0 -0
- data/Gemfile +20 -0
- data/Gemfile.lock +230 -0
- data/MIT-LICENSE +20 -0
- data/README.md +192 -0
- data/Rakefile +10 -0
- data/changelog.md +14 -0
- data/koyo-postgres-replication.gemspec +43 -0
- data/lib/koyo/repl/configuration.rb +110 -0
- data/lib/koyo/repl/data.rb +52 -0
- data/lib/koyo/repl/data_row.rb +100 -0
- data/lib/koyo/repl/database.rb +183 -0
- data/lib/koyo/repl/diagnostics.rb +93 -0
- data/lib/koyo/repl/event_handler_service.rb +58 -0
- data/lib/koyo/repl/install.rb +67 -0
- data/lib/koyo/repl/log.rb +98 -0
- data/lib/koyo/repl/mod.rb +41 -0
- data/lib/koyo/repl/postgres_server.rb +184 -0
- data/lib/koyo/repl/railtie.rb +39 -0
- data/lib/koyo/repl/templates/koyo_postgres_replication_config.txt +54 -0
- data/lib/koyo/repl/templates/koyo_repl_handler_service.txt +52 -0
- data/lib/koyo/repl/templates/koyo_repl_model_example.txt +19 -0
- data/lib/koyo/repl/version.rb +7 -0
- data/lib/koyo.rb +27 -0
- data/lib/koyo_postgres_replication.rb +5 -0
- metadata +168 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 78e654e0e390dedac52210540c5dafdb1c189b1bbb01968c8bd09b5bf5ee250f
|
4
|
+
data.tar.gz: 9dfdec20a290ae9d30ac91f7cfb404530053f883c9f12a57f678dd3665f4be8a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 97987ad66fbed7a4c14124de1776e771c3e54fb1c70b8f7fca6f588fc054dd858af38b753d2578e65aaefc25e9c048d86af49d6d3f6822ccbdadc7a76d42fbb9
|
7
|
+
data.tar.gz: d3fd6f95478bbf33830f7f8c85fcc3b99d9599987fea81cdd639f997d625bfd8fdfc99e9b4881da9ee6f703665d2019f9824ffc5ce38474828918c457abf82c6
|
data/.DS_Store
ADDED
Binary file
|
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--require spec_helper
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
Gemspec/RequiredRubyVersion:
|
2
|
+
Enabled: false
|
3
|
+
|
4
|
+
Metrics/AbcSize:
|
5
|
+
Enabled: false
|
6
|
+
|
7
|
+
Metrics/BlockLength:
|
8
|
+
Enabled: false
|
9
|
+
|
10
|
+
Metrics/ClassLength:
|
11
|
+
Enabled: false
|
12
|
+
|
13
|
+
Metrics/MethodLength:
|
14
|
+
Enabled: false
|
15
|
+
|
16
|
+
Layout/LineLength:
|
17
|
+
Max: 90
|
18
|
+
|
19
|
+
AllCops:
|
20
|
+
DisplayCopNames: true
|
21
|
+
Exclude:
|
22
|
+
- 'spec/dummy/config/**/*'
|
23
|
+
- 'spec/dummy/db/**/*'
|
data/.yardoc/checksums
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
lib/koyo.rb 0baca6fa8b0186eea9fee1eff1bba6853aa2e691
|
2
|
+
lib/koyo/repl/log.rb 811d21aa6f8843589b66da79a96a4e8b755fcc2d
|
3
|
+
lib/koyo/repl/mod.rb 92b54b56643b06fdc74adc65dc04460f631d4036
|
4
|
+
lib/koyo/repl/data.rb cb847585c0f439b9f645b5ebd7fa3b96e08b94ff
|
5
|
+
lib/koyo/repl/install.rb 9b19b7bb74d084882a1f726693bac2d693015daa
|
6
|
+
lib/koyo/repl/railtie.rb acb36cbbe1b386ece7567990945adecb6b6e5b80
|
7
|
+
lib/koyo/repl/version.rb 94239a7a9719294e13a38b878da1d328b44fb3f5
|
8
|
+
lib/koyo/repl/data_row.rb 321caf0b080621f2d319fa152948cbe864c77294
|
9
|
+
lib/koyo/repl/database.rb 39a304909ccf68d69d36c84e3036debd9f29f6ec
|
10
|
+
lib/koyo/repl/diagnostics.rb 03b91c5ea3dbe1e337f4427d8f3261ebedb45a70
|
11
|
+
lib/koyo/repl/configuration.rb 17858828602d696a584a9e3c239c0a5d94faa738
|
12
|
+
lib/koyo/repl/postgres_server.rb 2569fa642e55f6032e6ea73e9ab8762018d3cd6c
|
13
|
+
lib/koyo_postgres_replication.rb e3f4d905e3045b59f1f7c799d84cd9fde70eebd8
|
14
|
+
lib/koyo/repl/event_handler_service.rb 99b67bdcf3e0e92a04cd01bf8d05a4569b45d218
|
data/.yardoc/complete
ADDED
File without changes
|
Binary file
|
Binary file
|
data/.yardoc/proxy_types
ADDED
Binary file
|
data/Gemfile
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
source 'https://rubygems.org'
|
4
|
+
git_source(:github) { |repo| "https://github.com/#{repo}.git" }
|
5
|
+
|
6
|
+
# Specify your gem's dependencies in koyo_repl.gemspec
|
7
|
+
gemspec
|
8
|
+
|
9
|
+
gem 'rails', '~> 7.0'
|
10
|
+
gem 'rake', '~> 13.0'
|
11
|
+
gem 'pg', '~> 1.1'
|
12
|
+
|
13
|
+
group :development, :test do
|
14
|
+
gem 'debug', platforms: %i[mri mingw x64_mingw]
|
15
|
+
gem 'factory_bot_rails'
|
16
|
+
gem 'ffaker'
|
17
|
+
gem 'rspec-rails', '~> 6.0'
|
18
|
+
gem 'rubocop'
|
19
|
+
gem 'yard'
|
20
|
+
end
|
data/Gemfile.lock
ADDED
@@ -0,0 +1,230 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
koyo-postgres-replication (0.1.0.pre)
|
5
|
+
pg (~> 1.1)
|
6
|
+
rack (~> 2.0, >= 2.0.0)
|
7
|
+
rails (~> 7.0)
|
8
|
+
rake (~> 13.0)
|
9
|
+
|
10
|
+
GEM
|
11
|
+
remote: https://rubygems.org/
|
12
|
+
specs:
|
13
|
+
actioncable (7.0.7)
|
14
|
+
actionpack (= 7.0.7)
|
15
|
+
activesupport (= 7.0.7)
|
16
|
+
nio4r (~> 2.0)
|
17
|
+
websocket-driver (>= 0.6.1)
|
18
|
+
actionmailbox (7.0.7)
|
19
|
+
actionpack (= 7.0.7)
|
20
|
+
activejob (= 7.0.7)
|
21
|
+
activerecord (= 7.0.7)
|
22
|
+
activestorage (= 7.0.7)
|
23
|
+
activesupport (= 7.0.7)
|
24
|
+
mail (>= 2.7.1)
|
25
|
+
net-imap
|
26
|
+
net-pop
|
27
|
+
net-smtp
|
28
|
+
actionmailer (7.0.7)
|
29
|
+
actionpack (= 7.0.7)
|
30
|
+
actionview (= 7.0.7)
|
31
|
+
activejob (= 7.0.7)
|
32
|
+
activesupport (= 7.0.7)
|
33
|
+
mail (~> 2.5, >= 2.5.4)
|
34
|
+
net-imap
|
35
|
+
net-pop
|
36
|
+
net-smtp
|
37
|
+
rails-dom-testing (~> 2.0)
|
38
|
+
actionpack (7.0.7)
|
39
|
+
actionview (= 7.0.7)
|
40
|
+
activesupport (= 7.0.7)
|
41
|
+
rack (~> 2.0, >= 2.2.4)
|
42
|
+
rack-test (>= 0.6.3)
|
43
|
+
rails-dom-testing (~> 2.0)
|
44
|
+
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
45
|
+
actiontext (7.0.7)
|
46
|
+
actionpack (= 7.0.7)
|
47
|
+
activerecord (= 7.0.7)
|
48
|
+
activestorage (= 7.0.7)
|
49
|
+
activesupport (= 7.0.7)
|
50
|
+
globalid (>= 0.6.0)
|
51
|
+
nokogiri (>= 1.8.5)
|
52
|
+
actionview (7.0.7)
|
53
|
+
activesupport (= 7.0.7)
|
54
|
+
builder (~> 3.1)
|
55
|
+
erubi (~> 1.4)
|
56
|
+
rails-dom-testing (~> 2.0)
|
57
|
+
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
58
|
+
activejob (7.0.7)
|
59
|
+
activesupport (= 7.0.7)
|
60
|
+
globalid (>= 0.3.6)
|
61
|
+
activemodel (7.0.7)
|
62
|
+
activesupport (= 7.0.7)
|
63
|
+
activerecord (7.0.7)
|
64
|
+
activemodel (= 7.0.7)
|
65
|
+
activesupport (= 7.0.7)
|
66
|
+
activestorage (7.0.7)
|
67
|
+
actionpack (= 7.0.7)
|
68
|
+
activejob (= 7.0.7)
|
69
|
+
activerecord (= 7.0.7)
|
70
|
+
activesupport (= 7.0.7)
|
71
|
+
marcel (~> 1.0)
|
72
|
+
mini_mime (>= 1.1.0)
|
73
|
+
activesupport (7.0.7)
|
74
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
75
|
+
i18n (>= 1.6, < 2)
|
76
|
+
minitest (>= 5.1)
|
77
|
+
tzinfo (~> 2.0)
|
78
|
+
ast (2.4.2)
|
79
|
+
base64 (0.1.1)
|
80
|
+
builder (3.2.4)
|
81
|
+
concurrent-ruby (1.2.2)
|
82
|
+
crass (1.0.6)
|
83
|
+
date (3.3.3)
|
84
|
+
debug (1.8.0)
|
85
|
+
irb (>= 1.5.0)
|
86
|
+
reline (>= 0.3.1)
|
87
|
+
diff-lcs (1.5.0)
|
88
|
+
erubi (1.12.0)
|
89
|
+
factory_bot (6.2.1)
|
90
|
+
activesupport (>= 5.0.0)
|
91
|
+
factory_bot_rails (6.2.0)
|
92
|
+
factory_bot (~> 6.2.0)
|
93
|
+
railties (>= 5.0.0)
|
94
|
+
ffaker (2.21.0)
|
95
|
+
globalid (1.1.0)
|
96
|
+
activesupport (>= 5.0)
|
97
|
+
i18n (1.14.1)
|
98
|
+
concurrent-ruby (~> 1.0)
|
99
|
+
io-console (0.6.0)
|
100
|
+
irb (1.7.4)
|
101
|
+
reline (>= 0.3.6)
|
102
|
+
json (2.6.3)
|
103
|
+
language_server-protocol (3.17.0.3)
|
104
|
+
loofah (2.21.3)
|
105
|
+
crass (~> 1.0.2)
|
106
|
+
nokogiri (>= 1.12.0)
|
107
|
+
mail (2.8.1)
|
108
|
+
mini_mime (>= 0.1.1)
|
109
|
+
net-imap
|
110
|
+
net-pop
|
111
|
+
net-smtp
|
112
|
+
marcel (1.0.2)
|
113
|
+
method_source (1.0.0)
|
114
|
+
mini_mime (1.1.5)
|
115
|
+
minitest (5.19.0)
|
116
|
+
net-imap (0.3.7)
|
117
|
+
date
|
118
|
+
net-protocol
|
119
|
+
net-pop (0.1.2)
|
120
|
+
net-protocol
|
121
|
+
net-protocol (0.2.1)
|
122
|
+
timeout
|
123
|
+
net-smtp (0.3.3)
|
124
|
+
net-protocol
|
125
|
+
nio4r (2.5.9)
|
126
|
+
nokogiri (1.15.4-arm64-darwin)
|
127
|
+
racc (~> 1.4)
|
128
|
+
parallel (1.23.0)
|
129
|
+
parser (3.2.2.3)
|
130
|
+
ast (~> 2.4.1)
|
131
|
+
racc
|
132
|
+
pg (1.5.3)
|
133
|
+
racc (1.7.1)
|
134
|
+
rack (2.2.8)
|
135
|
+
rack-test (2.1.0)
|
136
|
+
rack (>= 1.3)
|
137
|
+
rails (7.0.7)
|
138
|
+
actioncable (= 7.0.7)
|
139
|
+
actionmailbox (= 7.0.7)
|
140
|
+
actionmailer (= 7.0.7)
|
141
|
+
actionpack (= 7.0.7)
|
142
|
+
actiontext (= 7.0.7)
|
143
|
+
actionview (= 7.0.7)
|
144
|
+
activejob (= 7.0.7)
|
145
|
+
activemodel (= 7.0.7)
|
146
|
+
activerecord (= 7.0.7)
|
147
|
+
activestorage (= 7.0.7)
|
148
|
+
activesupport (= 7.0.7)
|
149
|
+
bundler (>= 1.15.0)
|
150
|
+
railties (= 7.0.7)
|
151
|
+
rails-dom-testing (2.2.0)
|
152
|
+
activesupport (>= 5.0.0)
|
153
|
+
minitest
|
154
|
+
nokogiri (>= 1.6)
|
155
|
+
rails-html-sanitizer (1.6.0)
|
156
|
+
loofah (~> 2.21)
|
157
|
+
nokogiri (~> 1.14)
|
158
|
+
railties (7.0.7)
|
159
|
+
actionpack (= 7.0.7)
|
160
|
+
activesupport (= 7.0.7)
|
161
|
+
method_source
|
162
|
+
rake (>= 12.2)
|
163
|
+
thor (~> 1.0)
|
164
|
+
zeitwerk (~> 2.5)
|
165
|
+
rainbow (3.1.1)
|
166
|
+
rake (13.0.6)
|
167
|
+
regexp_parser (2.8.1)
|
168
|
+
reline (0.3.7)
|
169
|
+
io-console (~> 0.5)
|
170
|
+
rexml (3.2.6)
|
171
|
+
rspec-core (3.12.2)
|
172
|
+
rspec-support (~> 3.12.0)
|
173
|
+
rspec-expectations (3.12.3)
|
174
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
175
|
+
rspec-support (~> 3.12.0)
|
176
|
+
rspec-mocks (3.12.6)
|
177
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
178
|
+
rspec-support (~> 3.12.0)
|
179
|
+
rspec-rails (6.0.3)
|
180
|
+
actionpack (>= 6.1)
|
181
|
+
activesupport (>= 6.1)
|
182
|
+
railties (>= 6.1)
|
183
|
+
rspec-core (~> 3.12)
|
184
|
+
rspec-expectations (~> 3.12)
|
185
|
+
rspec-mocks (~> 3.12)
|
186
|
+
rspec-support (~> 3.12)
|
187
|
+
rspec-support (3.12.1)
|
188
|
+
rubocop (1.56.0)
|
189
|
+
base64 (~> 0.1.1)
|
190
|
+
json (~> 2.3)
|
191
|
+
language_server-protocol (>= 3.17.0)
|
192
|
+
parallel (~> 1.10)
|
193
|
+
parser (>= 3.2.2.3)
|
194
|
+
rainbow (>= 2.2.2, < 4.0)
|
195
|
+
regexp_parser (>= 1.8, < 3.0)
|
196
|
+
rexml (>= 3.2.5, < 4.0)
|
197
|
+
rubocop-ast (>= 1.28.1, < 2.0)
|
198
|
+
ruby-progressbar (~> 1.7)
|
199
|
+
unicode-display_width (>= 2.4.0, < 3.0)
|
200
|
+
rubocop-ast (1.29.0)
|
201
|
+
parser (>= 3.2.1.0)
|
202
|
+
ruby-progressbar (1.13.0)
|
203
|
+
thor (1.2.2)
|
204
|
+
timeout (0.4.0)
|
205
|
+
tzinfo (2.0.6)
|
206
|
+
concurrent-ruby (~> 1.0)
|
207
|
+
unicode-display_width (2.4.2)
|
208
|
+
websocket-driver (0.7.6)
|
209
|
+
websocket-extensions (>= 0.1.0)
|
210
|
+
websocket-extensions (0.1.5)
|
211
|
+
yard (0.9.34)
|
212
|
+
zeitwerk (2.6.11)
|
213
|
+
|
214
|
+
PLATFORMS
|
215
|
+
arm64-darwin-22
|
216
|
+
|
217
|
+
DEPENDENCIES
|
218
|
+
debug
|
219
|
+
factory_bot_rails
|
220
|
+
ffaker
|
221
|
+
koyo-postgres-replication!
|
222
|
+
pg (~> 1.1)
|
223
|
+
rails (~> 7.0)
|
224
|
+
rake (~> 13.0)
|
225
|
+
rspec-rails (~> 6.0)
|
226
|
+
rubocop
|
227
|
+
yard
|
228
|
+
|
229
|
+
BUNDLED WITH
|
230
|
+
2.4.18
|
data/MIT-LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright 2023 Ben Wiseley
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,192 @@
|
|
1
|
+
# Koyo::Postgres::Replication
|
2
|
+
|
3
|
+
## Replcation slots
|
4
|
+
|
5
|
+
This gem tries to simplify dealing with a `replication slot` in Postgres.
|
6
|
+
|
7
|
+
### What is a replication slot?
|
8
|
+
|
9
|
+
Please see [the wiki page that discusses replication
|
10
|
+
slots](https://github.com/wiseleyb/koyo-postgres-replication/wiki/What-is-replication%3F)
|
11
|
+
|
12
|
+
### Why would you use this?
|
13
|
+
|
14
|
+
Example: You have a no-sql store (like Elastic-Search) that needs to be in-sync
|
15
|
+
with your database. You could do this "the Rails way" with `after-save` type
|
16
|
+
patterns. You could do this with a service type architecture that updates
|
17
|
+
things. But if you have non-rails teams updating data, or ever update the
|
18
|
+
database via SQL, this becomes more complex, duplicates delicate/error-prone
|
19
|
+
work between teams, or just isn't possible. Going the microservice approach
|
20
|
+
also isn't ideal for performance reasons (you never want to updating a really
|
21
|
+
busy DB via API).
|
22
|
+
|
23
|
+
Please see [the
|
24
|
+
wiki](https://github.com/wiseleyb/koyo-postgres-replication/wiki/What-is-replication%3F)
|
25
|
+
for more discussion on this.
|
26
|
+
|
27
|
+
## Quick Start
|
28
|
+
|
29
|
+
You need to configure Postgres for this first. This isn't enabled by default.
|
30
|
+
See [Configuring Postgres for Replication in the
|
31
|
+
wiki](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Configuring-Postgres-for-Replication)
|
32
|
+
|
33
|
+
Add to Gemfile:
|
34
|
+
|
35
|
+
```
|
36
|
+
gem 'koyo-postgres-replication',
|
37
|
+
git: 'https://github.com/wiseleyb/koyo-postgres-replication',
|
38
|
+
require: 'koyo'
|
39
|
+
```
|
40
|
+
|
41
|
+
Then:
|
42
|
+
|
43
|
+
```
|
44
|
+
bundle install
|
45
|
+
bundle exec rake koyo:repl:install
|
46
|
+
```
|
47
|
+
|
48
|
+
This adds files:
|
49
|
+
|
50
|
+
* `config/initializers/koyo_postgres_replication_config.rb`: which is commented
|
51
|
+
on various ways to override various settings.
|
52
|
+
* `app/models/koyo_repl_handler_service.rb`: which has call backs for every
|
53
|
+
replication event, log events, and errors. This file is also heavily
|
54
|
+
commented.
|
55
|
+
* `app/models/koyo_repl_model_example.rb`: is an example of how to add
|
56
|
+
replication monitoring on a model level. You can delete this file - it's just
|
57
|
+
there for a simple example.
|
58
|
+
|
59
|
+
### Run Diagnostics
|
60
|
+
|
61
|
+
You need to create a replication slot for each database. This requires admin
|
62
|
+
access. In
|
63
|
+
[config](https://github.com/wiseleyb/koyo-postgres-replication/blob/main/lib/koyo/repl/templates/koyo_postgres_replication_config.txt)
|
64
|
+
you can setup a separate `config/database.yml` connection to limit admin level
|
65
|
+
postgres access if your company prefers that.
|
66
|
+
|
67
|
+
```
|
68
|
+
# runs basic diagnostics - look for Error in this list and fix any issues
|
69
|
+
bundle exec rake koyo:repl:diagnostics
|
70
|
+
```
|
71
|
+
|
72
|
+
Which will output something like:
|
73
|
+
|
74
|
+
```
|
75
|
+
--------------------------------------------------------------------------------
|
76
|
+
Koyo::Repl::Diagnostic
|
77
|
+
source=KoyoReplication logid=f3a8f68d3e level=info message=Init: Finding models that support koyo_repl_handler
|
78
|
+
source=KoyoReplication logid=410b6fd0d0 level=info message=Init: ignoring model SchemaMigration
|
79
|
+
source=KoyoReplication logid=b429e47251 level=info message=Init: ignoring model ArInternalMetadatum
|
80
|
+
source=KoyoReplication logid=6532a8ec76 level=info message=Init: registering handler ["users", "User"]
|
81
|
+
Config settings:
|
82
|
+
auto_create_replication_slot: true
|
83
|
+
config_prefix: KOYO_REPL
|
84
|
+
db_conn:
|
85
|
+
slot: koyo_repl_rei_postgres_replication_development_development
|
86
|
+
sql_delay: 1
|
87
|
+
test_mode: false
|
88
|
+
Replication slot exists: true
|
89
|
+
Registered tables:
|
90
|
+
users: User
|
91
|
+
Can connect to db: true
|
92
|
+
Connection adapter: PostgreSQL
|
93
|
+
Wal Level (should be 'logical'): logical
|
94
|
+
Can access replication slot: true
|
95
|
+
Replication slot count: 0
|
96
|
+
--------------------------------------------------------------------------------
|
97
|
+
```
|
98
|
+
|
99
|
+
If there are errors you'll need to fix those first before running the server.
|
100
|
+
|
101
|
+
### Run the server
|
102
|
+
|
103
|
+
```
|
104
|
+
bundle exec rake koyo:repl:run_server
|
105
|
+
```
|
106
|
+
|
107
|
+
Now - when you create/update/delete data in the configured database you should
|
108
|
+
be getting callbacks in
|
109
|
+
`app/models/koyo_repl_handler_service.rb#koyo_handle_all_replication(row)` and
|
110
|
+
in any model `app/models/{some-model}#handle_replication(row)` that implements
|
111
|
+
callbacks.
|
112
|
+
|
113
|
+
# Row data in callbacks
|
114
|
+
|
115
|
+
See [wiki page on DataRow for raw data examples of what you get from
|
116
|
+
Postgres](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Koyo::Repl::DataRow-data-spec)
|
117
|
+
|
118
|
+
# Processing callbacks
|
119
|
+
|
120
|
+
Both `koyo_repl_handler_service#koyo_handle_all_replication(row)` and
|
121
|
+
`{some-model}#handle_replication(row)` need to be REALLY fast. You shouldn't do
|
122
|
+
database updates from this code (or risk infinite loops) and, if you're doing
|
123
|
+
something like updating an API you should async that via Sidekiq, ActiveJob,
|
124
|
+
etc. Keep in mind that, if you're running multiple Sidekiq servers that things
|
125
|
+
might not be processed in the same order. Please see the [wiki page on
|
126
|
+
processing
|
127
|
+
callbacks](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Processing-callbacks)
|
128
|
+
for more on this.
|
129
|
+
|
130
|
+
# Testing
|
131
|
+
|
132
|
+
The following assumes you're using rspec (open to PRs supporting other test
|
133
|
+
frameworks though - I just don't use those)
|
134
|
+
|
135
|
+
By default (and definitely the fastest way) specs run inside
|
136
|
+
transactions that are rolled back after each spec runs. You need to use
|
137
|
+
[Database Cleaner](https://github.com/DatabaseCleaner/database_cleaner)
|
138
|
+
truncation approach to test this stuff. Please see the wiki on
|
139
|
+
[testing](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Testing)
|
140
|
+
for what's needed if you want to run replication slot tests.
|
141
|
+
|
142
|
+
# Sample apps
|
143
|
+
|
144
|
+
These are simple Rails version-specific demo apps with Docker files
|
145
|
+
|
146
|
+
* [Rails 7 Example](https://github.com/wiseleyb/rei-postgres-replication/tree/rails-7-example)
|
147
|
+
* Rails 6 Example: coming soon
|
148
|
+
* Rails 5 Example: coming soon
|
149
|
+
* Rails 4 Example: coming soon
|
150
|
+
* Rails 3 Example: probably won't do
|
151
|
+
|
152
|
+
# Technical
|
153
|
+
|
154
|
+
See wiki page on [creating this
|
155
|
+
gem](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Creating-this-GEM)
|
156
|
+
if you're interested how to do gems like these.
|
157
|
+
|
158
|
+
## Working with the gem
|
159
|
+
|
160
|
+
See wiki page on [working with this
|
161
|
+
gem](https://github.com/wiseleyb/koyo-postgres-replication/wiki/Developing-on-this-GEM)
|
162
|
+
for basics of debugging and working with.
|
163
|
+
|
164
|
+
## Working with replication slots
|
165
|
+
|
166
|
+
See
|
167
|
+
[koyo::repl::database](https://github.com/wiseleyb/koyo-postgres-replication/blob/main/lib/koyo/repl/database.rb)
|
168
|
+
for sql examples on how to interact with replication slots.
|
169
|
+
|
170
|
+
## Yard Doc
|
171
|
+
|
172
|
+
Cheat sheets:
|
173
|
+
* https://gist.github.com/chetan/1827484
|
174
|
+
* https://kapeli.com/cheat_sheets/Yard.docset/Contents/Resources/Documents/index
|
175
|
+
|
176
|
+
Build yard docs: `yard`
|
177
|
+
View docs: `yard server` then open https://localhost:8808
|
178
|
+
|
179
|
+
# Contributing
|
180
|
+
|
181
|
+
TODO: update
|
182
|
+
|
183
|
+
* follow github guide
|
184
|
+
* run/add specs
|
185
|
+
* run rubocop
|
186
|
+
* add/run yard
|
187
|
+
|
188
|
+
# TODO
|
189
|
+
|
190
|
+
* add monitoring helpers
|
191
|
+
* add support for multiple primary keys
|
192
|
+
* add badges like in https://raw.githubusercontent.com/rubocop/rubocop/master/README.md
|
data/Rakefile
ADDED
data/changelog.md
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
# Koyo Postgres Replication Changelog
|
2
|
+
|
3
|
+
## 0.1.0.pre
|
4
|
+
|
5
|
+
- rails 7 plugin
|
6
|
+
- postgres replication monitor
|
7
|
+
- install script
|
8
|
+
- documentation
|
9
|
+
- basic specs
|
10
|
+
- example rails project
|
11
|
+
- diagnostic tools
|
12
|
+
- utility class for working with replication slots
|
13
|
+
- functional - but not tested in real life yet
|
14
|
+
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'lib/koyo'
|
4
|
+
|
5
|
+
burl = 'https://github.com/wiseleyb'
|
6
|
+
url = "#{burl}/koyo-postgres-replication"
|
7
|
+
|
8
|
+
Gem::Specification.new do |spec|
|
9
|
+
spec.name = 'koyo-postgres-replication'
|
10
|
+
spec.version = Koyo::Repl::VERSION
|
11
|
+
spec.authors = ['Ben Wiseley']
|
12
|
+
spec.email = ['wiseleyb@gmail.com']
|
13
|
+
spec.homepage = burl
|
14
|
+
spec.summary = 'Postgres Replication'
|
15
|
+
spec.description = 'Simple Postgres replication helper'
|
16
|
+
spec.license = 'MIT'
|
17
|
+
|
18
|
+
spec.required_ruby_version = '>= 2.7.0'
|
19
|
+
|
20
|
+
# spec.metadata["allowed_push_host"] = ''
|
21
|
+
spec.metadata['changelog_uri'] = "#{url}/changelog.md"
|
22
|
+
spec.metadata['homepage_uri'] = spec.homepage
|
23
|
+
spec.metadata['source_code_uri'] = url
|
24
|
+
|
25
|
+
spec.files = Dir.chdir(__dir__) do
|
26
|
+
`git ls-files -z`.split("\x0").reject do |f|
|
27
|
+
(File.expand_path(f) ==
|
28
|
+
__FILE__) ||
|
29
|
+
f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor])
|
30
|
+
end
|
31
|
+
end
|
32
|
+
spec.bindir = 'bin'
|
33
|
+
spec.require_paths = ['lib']
|
34
|
+
|
35
|
+
spec.add_dependency 'pg', '~> 1.1'
|
36
|
+
spec.add_dependency 'rack', '~> 2.0', '>= 2.0.0'
|
37
|
+
spec.add_dependency 'rails', '~> 7.0'
|
38
|
+
spec.add_dependency 'rake', '~> 13.0'
|
39
|
+
spec.add_development_dependency 'rspec-rails', '~> 3.1'
|
40
|
+
spec.add_development_dependency 'yard', '~> 0.9'
|
41
|
+
|
42
|
+
spec.has_rdoc = 'yard'
|
43
|
+
end
|