koyo-postgres-replication 0.1.0.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,98 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Include this for logging help: include Koyo::Repl::Log
4
+ module Koyo
5
+ module Repl
6
+ # Log helper tools. Include this in your class to use
7
+ module Log
8
+ LOG_LEVELS = %w[debug info warn error fatal].freeze
9
+
10
+ # Run when "include Koyo::Repl::Log" is called from including class
11
+ def self.included(base)
12
+ base.extend(ClassMethods)
13
+
14
+ # Creates helper methods based on LOG_LEVELS
15
+ LOG_LEVELS.each do |lvl|
16
+ define_method "log_repl_#{lvl}" do |message, data = {}|
17
+ self.class.log_repl(message, data, log_level: lvl.to_sym)
18
+ end
19
+ end
20
+ end
21
+
22
+ # Instance level methods
23
+ module ClassMethods
24
+ # Creates helper methods based on LOG_LEVELS
25
+ LOG_LEVELS.each do |lvl|
26
+ define_method "log_repl_#{lvl}" do |message, data = {}|
27
+ log_repl(message, data, log_level: lvl.to_sym)
28
+ end
29
+ end
30
+
31
+ # Log message and a hash with level
32
+ # @param [String] message Arbitrary string to output
33
+ # @param [Hash] data to add to log message. if :err is included this
34
+ # will parse message and backtrace
35
+ # @param [Symbol] log_level defaults to :info (LOG_LEVELS contains
36
+ # other options
37
+ def log_repl(message, data = {}, log_level: :info)
38
+ return if message.blank?
39
+ return if Koyo::Repl.config.disable_logging
40
+
41
+ err = data.delete(:err)
42
+ if err
43
+ data[:err_message] ||= err.message
44
+ data[:err_backtrace] ||= err.backtrace.join("\n")
45
+ end
46
+ data[:message] ||= message
47
+
48
+ log_repl_hash(data, log_level)
49
+ end
50
+
51
+ # Logs messages with formating like:
52
+ # source=KoyoReplication
53
+ # logid=short guid for uniqueness if needed
54
+ # level=log_level
55
+ # {key}={value} data being logged
56
+ # @param [Hash] hash keys/values being logged
57
+ # @param [Symbol] log_level see LOG_LEVELS constant for options
58
+ def log_repl_hash(hash, log_level)
59
+ return if Koyo::Repl.config.disable_logging
60
+
61
+ logid = SecureRandom.hex(5)
62
+
63
+ hash.each do |k, v|
64
+ log_repl_key_value(logid, log_level, k, v)
65
+ end
66
+ end
67
+
68
+ # Logs message with formating like:
69
+ # source=KoyoReplication
70
+ # logid=short guid for uniqueness if needed
71
+ # level=log_level
72
+ # {key}={value} data being logged
73
+ # @param [String] logid used incase uniqueness is required
74
+ # @param [Symbol] log_level see contant LOG_LEVELS for options
75
+ # @param [String] key any string to log
76
+ # @param [String] val any string to log
77
+ def log_repl_key_value(logid, log_level, key, val)
78
+ msg = "source=KoyoReplication logid=#{logid} level=#{log_level} "\
79
+ "#{key}=#{val}"
80
+ puts msg unless Rails.env.test? # don't pollute test output
81
+ check_log_level(log_level)
82
+ Rails.logger.send(log_level, msg)
83
+ Koyo::Repl::EventHandlerService.koyo_log_event(msg, log_level)
84
+ Koyo::Repl::EventHandlerService.send("koyo_log_event_#{log_level}",
85
+ msg)
86
+ end
87
+
88
+ # Validates log_level is in contant LOG_LEVELS
89
+ # @param log_level to validate
90
+ def check_log_level(log_level)
91
+ return if LOG_LEVELS.include?(log_level.to_s)
92
+
93
+ raise "Invalid logger level. Valid options are #{LOG_LEVELS}"
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Koyo
4
+ module Repl
5
+ # Used to support model/table level replication handlers
6
+ #
7
+ # Example:
8
+ # class User < ApplicationRecord
9
+ # include Koyo::Repl::Mod
10
+ #
11
+ # # register method for replication
12
+ # koyo_repl_handler :handle_replication
13
+ #
14
+ # # This is called when a row is created/updated/deleted
15
+ # # You don't want to do DB updates from this or you will likely
16
+ # # create an infinite loop
17
+ # def self.handle_replication(row)
18
+ # msg = [
19
+ # '*' * 80,
20
+ # row.to_yaml,
21
+ # '*' * 80
22
+ # ]
23
+ # puts msg
24
+ # end
25
+ # end
26
+ module Mod
27
+ def self.included(base)
28
+ base.extend(ClassMethods)
29
+ end
30
+
31
+ # Enables `koyo_repl_handler :handle_replication`
32
+ module ClassMethods
33
+ attr_accessor :koyo_repl_handler_method
34
+
35
+ def koyo_repl_handler(method_name)
36
+ @koyo_repl_handler_method = method_name
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,184 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Koyo
4
+ module Repl
5
+ # Monitors a postgres replication slot and fires off events
6
+ # for monitoring to models and a catch-all class
7
+ class PostgresServer
8
+ # includes log helpers
9
+ include Koyo::Repl::Log
10
+
11
+ class << self
12
+ attr_accessor :tables # cache of class level :tables
13
+ end
14
+
15
+ attr_accessor :tables, # classes that implement koyo_repl_handler(row)
16
+ :test_mode, # when true - only peeks at slot
17
+ :errs # collects error messages - these are sent to log
18
+
19
+ attr_writer :tick_tock # just outputs something in logs every minute
20
+
21
+ # Method name to look for in models that support this
22
+ # @see Model callbacks
23
+ # https://github.com/wiseleyb/koyo-postgres-replication/wiki/Model-call-backs
24
+ TABLE_METHOD_NAME = :koyo_repl_handler
25
+
26
+ # Runs the server. You should only be running ONE of these
27
+ # servers at a time.
28
+ def self.run!
29
+ new.run!
30
+ end
31
+
32
+ # Initialize server: checks for basics and fails if things
33
+ # aren't setup
34
+ def initialize
35
+ @test_mode = Koyo::Repl.config.test_mode
36
+ if Koyo::Repl.config.auto_create_replication_slot
37
+ Koyo::Repl::Database.create_replication_slot!
38
+ end
39
+ @tables = Koyo::Repl::PostgresServer.tables_that_handle_koyo_replication
40
+ @errs = []
41
+ @tick_tock = 0
42
+ raise "Can't run server: #{@errs.join('; ')}" unless can_run?
43
+ end
44
+
45
+ # Runs the server. You should only be running ONE of these
46
+ # servers at a time.
47
+ def run!
48
+ # This allows us to catch ctrl-c and exit
49
+ trap('SIGINT') { throw :done }
50
+
51
+ catch(:done) do
52
+ check
53
+ sleep Koyo::Repl.config.sql_delay
54
+ tick_tock
55
+ run!
56
+ # Possibly fatal errors
57
+ rescue ActiveRecord::StatementInvalid => e
58
+ if e.cause.exception.is_a?(PG::ConnectionBad)
59
+ Koyo::Repl::EventHandlerService.koyo_error(e)
60
+ msg = "SHUTTING DOWN. Fatal Error in ReplPostgresServer: #{e.message}"
61
+ log_repl_fatal(msg, err: e)
62
+ else
63
+ log_recoverable_error(e)
64
+ run!
65
+ end
66
+ rescue StandardError => e
67
+ log_recoverable_error(e)
68
+ run!
69
+ end
70
+ end
71
+
72
+ # Handles erros that aren't fatal. Calls back to
73
+ # Koyo::Repl::Log@log_repl_error which calls back
74
+ # to KoyoReplHandlerServer@log_repl_error
75
+ def log_recoverable_error(err)
76
+ Koyo::Repl::EventHandlerService.koyo_error(err)
77
+ msg = "Error in ReplPostgresServer: #{err.message}"
78
+ log_repl_error(msg, err: err)
79
+ sleep Koyo::Repl.config.sql_delay
80
+ end
81
+
82
+ # Basic heart beat ping to allow you to see the server is still
83
+ # running. Pings every 60 seconds
84
+ def tick_tock
85
+ @tick_tock += 1
86
+ return unless @tick_tock > 59
87
+
88
+ log_repl_info("tick tock: #{Time.now}")
89
+ @tick_tock = 0
90
+ end
91
+
92
+ # Does a single check of the replication slot
93
+ # If test_mode=true uses peek, which will
94
+ # leave data in the replication slot (for testing/debugging)
95
+ def check
96
+ read_sql_results.each do |sql_res|
97
+ rows = Koyo::Repl::Data.new(sql_res).rows # returns ReplDataRow
98
+ rows.each do |row|
99
+ check_row(row)
100
+ end
101
+ end
102
+ end
103
+
104
+ # Reads data from the replication slot
105
+ # Handles test_mode (so will only peek if true)
106
+ def read_sql_results
107
+ if test_mode
108
+ Koyo::Repl::Database.peek_slot
109
+ else
110
+ Koyo::Repl::Database.read_slot!
111
+ end
112
+ end
113
+
114
+ # Processes a row from the replication slot
115
+ # @param row Koyo::Repl::DataRow
116
+ # @see For details on row
117
+ # https://github.com/wiseleyb/koyo-postgres-replication/wiki/Koyo::Repl::DataRow-data-spec
118
+ def check_row(row)
119
+ log_repl_debug(row.to_yaml)
120
+ # catch all for all events (allows rails project to use this
121
+ # instead of models
122
+ Koyo::Repl::EventHandlerService.koyo_handle_all_replication(row)
123
+
124
+ return unless tables.include?(row.table)
125
+
126
+ # handle model callbacks
127
+ klass = tables[row.table].constantize
128
+ mname = klass.send("#{TABLE_METHOD_NAME}_method")
129
+ klass.send(mname, row)
130
+ rescue StandardError => e
131
+ Koyo::Repl::EventHandlerService.koyo_error(e)
132
+ log_repl_error('Unexpected Error in ReplServer.check', err: e)
133
+ end
134
+
135
+ # Checks basics to see if we can run
136
+ # Logs errors (should be visible in whatever is running the server
137
+ # @return true can run when true or false (can't run)
138
+ def can_run?
139
+ @errs = []
140
+
141
+ # check if replication slot is setup
142
+ unless Koyo::Repl::Database.replication_slot_exists?
143
+ errs << "Error: Replication Slot doesn't exist. "\
144
+ 'See koyo-postgres-replication gem for how to set this up.'
145
+ end
146
+
147
+ # if there were any errors - let user know we're shutting down
148
+ errs << 'Shutting down' unless errs.empty?
149
+
150
+ errs.each { |msg| log_repl_error(msg) }
151
+
152
+ errs.empty?
153
+ end
154
+
155
+ # Finds all models that that implement 'self.koyo_repl_handler'
156
+ # This is only run once - during server spin up
157
+ def self.tables_that_handle_koyo_replication
158
+ return tables if tables.present?
159
+
160
+ log_repl_info('Init: Finding models that support '\
161
+ "#{TABLE_METHOD_NAME}")
162
+ tables = {}
163
+ ActiveRecord::Base.connection.tables.map do |model|
164
+ klass_name = model.capitalize.singularize.camelize
165
+ klass = klass_name.constantize
166
+ next unless klass.methods.include?(TABLE_METHOD_NAME)
167
+
168
+ tables[klass.table_name] = klass_name
169
+ rescue NameError # filters out stuff like SchemaMigration
170
+ log_repl_info("Init: ignoring model #{klass_name}")
171
+ rescue StandardError => e
172
+ Koyo::Repl::EventHandlerService.koyo_error(e)
173
+ log_repl_error('Unexpected Error in '\
174
+ 'ReplServer.tables_that_handle_koyo_replication',
175
+ err: e)
176
+ end
177
+ tables.each do |t|
178
+ log_repl_info("Init: registering handler #{t}")
179
+ end
180
+ tables
181
+ end
182
+ end
183
+ end
184
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails'
4
+
5
+ module Koyo
6
+ module Repl
7
+ # Adds rake tasks accessible to Rails project
8
+ class Railtie < Rails::Railtie
9
+ railtie_name :koyo_repl
10
+
11
+ rake_tasks do
12
+ namespace :koyo do
13
+ namespace :repl do
14
+ desc 'Diagnostics: Basic setup/state information'
15
+ task diagnostics: :environment do
16
+ puts ''
17
+ puts '-' * 80
18
+ puts 'Koyo::Repl::Diagnostic'
19
+ puts Koyo::Repl::Diagnostics.new.rake_info.join("\n")
20
+ puts '-' * 80
21
+ puts ''
22
+ end
23
+
24
+ desc 'Install templates unless they already exist'
25
+ task install: :environment do
26
+ Koyo::Repl::Install.copy!
27
+ end
28
+
29
+ desc 'Process replication slot events - only run this server ONCE'
30
+ task run_server: :environment do
31
+ puts 'Running Koyo::Repl::PostgresServer.run!'
32
+ Koyo::Repl::PostgresServer.run!
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,54 @@
1
+ Koyo::Repl.configure do |config|
2
+ # Create the replication slot if it doesn't exist. Defaults to true
3
+ # You can set this with an ENV as well:
4
+ # KOYO_REPL_AUTO_CREATE_REPLICATION_SLOT
5
+ # config.auto_create_replication_slot = true
6
+
7
+ # Allows you to override the prefix used if you're using ENV to configure
8
+ # things. Defaults to KOYO_REPL
9
+ # config.config_prefix = 'KOYO_REPL'
10
+
11
+ # DB connection name in config/database.yml. Defaults to Rails.env
12
+ # (so standard connection on most rails app). We add this because you need
13
+ # admin priveleges to use replication and some companies have problems with
14
+ # this. Whatever this is called it will have Rails.env tacked on so if it's
15
+ # replication - the connection would be "replciation_#{Rails.env}"
16
+ # You can set this with an ENV as well:
17
+ # KOYO_REPL_DATABASE_NAME
18
+ # config.database_name = 'replication'
19
+
20
+ # Disable logging. Not recommended.
21
+ # You can set this with an ENV as well:
22
+ # KOYO_REPL_DISABLE_LOGGING
23
+ # config.disable_logging = true
24
+
25
+ # Determines the name of this replication slot. Defaults to
26
+ # koyo_repl_{Rails.env}.
27
+ # You can check replication slots that exist with:
28
+ #
29
+ # select slot_name
30
+ # from pg_replication_slots
31
+ # where
32
+ # and plugin = 'wal2json'
33
+ # You can set this with an ENV as well:
34
+ # KOYO_REPL_SLOT
35
+ # config.slot = "koyo_repl_#{Rails.env}"
36
+
37
+ # Time to wait before checking Replication Slot again in seconds
38
+ # Note: that if there 10,000 things on the replciation-queue it will
39
+ # process all of those as fast as possible, then pause for this many
40
+ # seconds before re-checking the replication-queue
41
+ # You can set this with an ENV as well:
42
+ # KOYO_REPL_SQL_DELAY
43
+ # config.sql_delay = 1
44
+
45
+ # When true we only "peek" the replication slot
46
+ # Peek (when this is false):
47
+ # leaves the data on the postgres-replication queue
48
+ # Read (when this is true):
49
+ # removes data from the postgres-replication queue
50
+ # Defaults to false
51
+ # You can set this with an ENV as well:
52
+ # KOYO_REPL_TEST_MODE
53
+ # config.test_mode = false
54
+ end
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+ # You can override these methods to do things like ping Slack, retart
3
+ # crashed things, handle errors, handle logging, handle all replicaiton events
4
+ # (instead of doing this in models)
5
+ class KoyoReplHandlerService < Koyo::Repl::EventHandlerService
6
+ class << self
7
+ # This is called for every create/update/delete action called on any table
8
+ # This can be used if you want to process all replication in one place
9
+ # instead of using the model level mixins. See README
10
+ # For an app with a lot of DB churn you want this to be as fast as possible
11
+ # You don't want do API calls from here, and shouldn't do DB updates from
12
+ # here (but if you do, be super careful of infinite loops)
13
+ # def koyo_handle_all_replication(row)
14
+ # case row.table
15
+ # when 'users'
16
+ # # do something with user like
17
+ # # ElasticSearchUpdaterServer.perform_async(row.id)
18
+ # # or grab the user record
19
+ # # user = User.find(row.id)
20
+ # end
21
+ # super
22
+ # end
23
+ def koyo_handle_all_replication(row); end
24
+
25
+ # Called whenever an error is raised in Koyo::Repl code
26
+ # Examples would be to log to Slack, or restart something that's crashed
27
+ def koyo_error(err); end
28
+
29
+ # log_level: :debug, :info, :warn, :error, :fatal
30
+ # Example of message
31
+ # source=KoyoReplication logid=d7f1f0bb2a
32
+ # message=Init: Finding models that support koyo_repl_handler
33
+ # You can use this as a catch all for any log event or use methods
34
+ # below if that's easier. You could use this for monitoring type activites
35
+ def koyo_log_event(message, log_level); end
36
+
37
+ # Called whenever Rails.logger.debug is called from Koyo::Repl code
38
+ def koyo_log_event_debug(message); end
39
+
40
+ # Called whenever Rails.logger.debug is called from Koyo::Repl code
41
+ def koyo_log_event_info(message); end
42
+
43
+ # Called whenever Rails.logger.debug is called from Koyo::Repl code
44
+ def koyo_log_event_warn(message); end
45
+
46
+ # Called whenever Rails.logger.debug is called from Koyo::Repl code
47
+ def koyo_log_event_error(message); end
48
+
49
+ # Called whenever Rails.logger.debug is called from Koyo::Repl code
50
+ def koyo_log_event_fatal(message); end
51
+ end
52
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Basic model example
4
+ class KoyoReplModelExample < ApplicationRecord
5
+ include Koyo::Repl::Mod
6
+
7
+ # register method for replication
8
+ koyo_repl_handler :handle_replication
9
+
10
+ # This is called when a row is created/updated/deleted
11
+ # You don't want to do DB updates from this or you will likely
12
+ # create an infinite loop
13
+ # This needs to be REALLY fast if you have a lot of db traffic
14
+ # For example: if you're doing something like calling an API from this
15
+ # method you should async it (put it in Sidekiq, ActiveJob, etc)
16
+ def self.handle_replication(row)
17
+ puts row.to_yaml
18
+ end
19
+ end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Koyo
4
+ module Repl
5
+ VERSION = '0.1.0.pre'
6
+ end
7
+ end
data/lib/koyo.rb ADDED
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Koyo
4
+ # Main replication namespace
5
+ module Repl
6
+ require_relative 'koyo/repl/configuration'
7
+ require_relative 'koyo/repl/data'
8
+ require_relative 'koyo/repl/database'
9
+ require_relative 'koyo/repl/data_row'
10
+ require_relative 'koyo/repl/diagnostics'
11
+ require_relative 'koyo/repl/event_handler_service'
12
+ require_relative 'koyo/repl/install'
13
+ require_relative 'koyo/repl/log'
14
+ require_relative 'koyo/repl/mod'
15
+ require_relative 'koyo/repl/postgres_server'
16
+ require_relative 'koyo/repl/railtie' # if defined?(Rails)
17
+ require_relative 'koyo/repl/version'
18
+
19
+ def self.config
20
+ @config ||= Configuration.new
21
+ end
22
+
23
+ def self.configure
24
+ yield(config)
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Dummy mod for gemspec
4
+ module KoyoPostgresReplication
5
+ end