koyo-postgres-replication 0.1.0.pre

data/lib/koyo/repl/log.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# Include this for logging help: include Koyo::Repl::Log
+module Koyo
+  module Repl
+    # Log helper tools. Include this in your class to use
+    module Log
+      LOG_LEVELS = %w[debug info warn error fatal].freeze
+
+      # Run when "include Koyo::Repl::Log" is called from including class
+      def self.included(base)
+        base.extend(ClassMethods)
+
+        # Creates helper methods based on LOG_LEVELS
+        LOG_LEVELS.each do |lvl|
+          define_method "log_repl_#{lvl}" do |message, data = {}|
+            self.class.log_repl(message, data, log_level: lvl.to_sym)
+          end
+        end
+      end
+
+      # Instance level methods
+      module ClassMethods
+        # Creates helper methods based on LOG_LEVELS
+        LOG_LEVELS.each do |lvl|
+          define_method "log_repl_#{lvl}" do |message, data = {}|
+            log_repl(message, data, log_level: lvl.to_sym)
+          end
+        end
+
+        # Log message and a hash with level
+        # @param [String] message Arbitrary string to output
+        # @param [Hash] data to add to log message. if :err is included this
+        # will parse message and backtrace
+        # @param [Symbol] log_level defaults to :info (LOG_LEVELS contains
+        # other options
+        def log_repl(message, data = {}, log_level: :info)
+          return if message.blank?
+          return if Koyo::Repl.config.disable_logging
+
+          err = data.delete(:err)
+          if err
+            data[:err_message] ||= err.message
+            data[:err_backtrace] ||= err.backtrace.join("\n")
+          end
+          data[:message] ||= message
+
+          log_repl_hash(data, log_level)
+        end
+
+        # Logs messages with formating like:
+        #   source=KoyoReplication
+        #   logid=short guid for uniqueness if needed
+        #   level=log_level
+        #   {key}={value} data being logged
+        # @param [Hash] hash keys/values being logged
+        # @param [Symbol] log_level see LOG_LEVELS constant for options
+        def log_repl_hash(hash, log_level)
+          return if Koyo::Repl.config.disable_logging
+
+          logid = SecureRandom.hex(5)
+
+          hash.each do |k, v|
+            log_repl_key_value(logid, log_level, k, v)
+          end
+        end
+
+        # Logs message with formating like:
+        #   source=KoyoReplication
+        #   logid=short guid for uniqueness if needed
+        #   level=log_level
+        #   {key}={value} data being logged
+        # @param [String] logid used incase uniqueness is required
+        # @param [Symbol] log_level see contant LOG_LEVELS for options
+        # @param [String] key any string to log
+        # @param [String] val any string to log
+        def log_repl_key_value(logid, log_level, key, val)
+          msg = "source=KoyoReplication logid=#{logid} level=#{log_level} "\
+                "#{key}=#{val}"
+          puts msg unless Rails.env.test? # don't pollute test output
+          check_log_level(log_level)
+          Rails.logger.send(log_level, msg)
+          Koyo::Repl::EventHandlerService.koyo_log_event(msg, log_level)
+          Koyo::Repl::EventHandlerService.send("koyo_log_event_#{log_level}",
+                                               msg)
+        end
+
+        # Validates log_level is in contant LOG_LEVELS
+        # @param log_level to validate
+        def check_log_level(log_level)
+          return if LOG_LEVELS.include?(log_level.to_s)
+
+          raise "Invalid logger level. Valid options are #{LOG_LEVELS}"
+        end
+      end
+    end
+  end
+end

data/lib/koyo/repl/mod.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Koyo
+  module Repl
+    # Used to support model/table level replication handlers
+    #
+    # Example:
+    #   class User < ApplicationRecord
+    #     include Koyo::Repl::Mod
+    #
+    #     # register method for replication
+    #     koyo_repl_handler :handle_replication
+    #
+    #     # This is called when a row is created/updated/deleted
+    #     # You don't want to do DB updates from this or you will likely
+    #     # create an infinite loop
+    #     def self.handle_replication(row)
+    #       msg = [
+    #         '*' * 80,
+    #         row.to_yaml,
+    #         '*' * 80
+    #       ]
+    #       puts msg
+    #     end
+    #   end
+    module Mod
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      # Enables `koyo_repl_handler :handle_replication`
+      module ClassMethods
+        attr_accessor :koyo_repl_handler_method
+
+        def koyo_repl_handler(method_name)
+          @koyo_repl_handler_method = method_name
+        end
+      end
+    end
+  end
+end

data/lib/koyo/repl/postgres_server.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module Koyo
+  module Repl
+    # Monitors a postgres replication slot and fires off events
+    # for monitoring to models and a catch-all class
+    class PostgresServer
+      # includes log helpers
+      include Koyo::Repl::Log
+
+      class << self
+        attr_accessor :tables # cache of class level :tables
+      end
+
+      attr_accessor :tables, # classes that implement koyo_repl_handler(row)
+                    :test_mode, # when true - only peeks at slot
+                    :errs # collects error messages - these are sent to log
+
+      attr_writer :tick_tock # just outputs something in logs every minute
+
+      # Method name to look for in models that support this
+      # @see Model callbacks
+      # https://github.com/wiseleyb/koyo-postgres-replication/wiki/Model-call-backs
+      TABLE_METHOD_NAME = :koyo_repl_handler
+
+      # Runs the server. You should only be running ONE of these
+      # servers at a time.
+      def self.run!
+        new.run!
+      end
+
+      # Initialize server: checks for basics and fails if things
+      # aren't setup
+      def initialize
+        @test_mode = Koyo::Repl.config.test_mode
+        if Koyo::Repl.config.auto_create_replication_slot
+          Koyo::Repl::Database.create_replication_slot!
+        end
+        @tables = Koyo::Repl::PostgresServer.tables_that_handle_koyo_replication
+        @errs = []
+        @tick_tock = 0
+        raise "Can't run server: #{@errs.join('; ')}" unless can_run?
+      end
+
+      # Runs the server. You should only be running ONE of these
+      # servers at a time.
+      def run!
+        # This allows us to catch ctrl-c and exit
+        trap('SIGINT') { throw :done }
+
+        catch(:done) do
+          check
+          sleep Koyo::Repl.config.sql_delay
+          tick_tock
+          run!
+        # Possibly fatal errors
+        rescue ActiveRecord::StatementInvalid => e
+          if e.cause.exception.is_a?(PG::ConnectionBad)
+            Koyo::Repl::EventHandlerService.koyo_error(e)
+            msg = "SHUTTING DOWN. Fatal Error in ReplPostgresServer: #{e.message}"
+            log_repl_fatal(msg, err: e)
+          else
+            log_recoverable_error(e)
+            run!
+          end
+        rescue StandardError => e
+          log_recoverable_error(e)
+          run!
+        end
+      end
+
+      # Handles erros that aren't fatal. Calls back to
+      # Koyo::Repl::Log@log_repl_error which calls back
+      # to KoyoReplHandlerServer@log_repl_error
+      def log_recoverable_error(err)
+        Koyo::Repl::EventHandlerService.koyo_error(err)
+        msg = "Error in ReplPostgresServer: #{err.message}"
+        log_repl_error(msg, err: err)
+        sleep Koyo::Repl.config.sql_delay
+      end
+
+      # Basic heart beat ping to allow you to see the server is still
+      # running. Pings every 60 seconds
+      def tick_tock
+        @tick_tock += 1
+        return unless @tick_tock > 59
+
+        log_repl_info("tick tock: #{Time.now}")
+        @tick_tock = 0
+      end
+
+      # Does a single check of the replication slot
+      # If test_mode=true uses peek, which will
+      # leave data in the replication slot (for testing/debugging)
+      def check
+        read_sql_results.each do |sql_res|
+          rows = Koyo::Repl::Data.new(sql_res).rows # returns ReplDataRow
+          rows.each do |row|
+            check_row(row)
+          end
+        end
+      end
+
+      # Reads data from the replication slot
+      # Handles test_mode (so will only peek if true)
+      def read_sql_results
+        if test_mode
+          Koyo::Repl::Database.peek_slot
+        else
+          Koyo::Repl::Database.read_slot!
+        end
+      end
+
+      # Processes a row from the replication slot
+      # @param row Koyo::Repl::DataRow
+      # @see For details on row
+      # https://github.com/wiseleyb/koyo-postgres-replication/wiki/Koyo::Repl::DataRow-data-spec
+      def check_row(row)
+        log_repl_debug(row.to_yaml)
+        # catch all for all events (allows rails project to use this
+        # instead of models
+        Koyo::Repl::EventHandlerService.koyo_handle_all_replication(row)
+
+        return unless tables.include?(row.table)
+
+        # handle model callbacks
+        klass = tables[row.table].constantize
+        mname = klass.send("#{TABLE_METHOD_NAME}_method")
+        klass.send(mname, row)
+      rescue StandardError => e
+        Koyo::Repl::EventHandlerService.koyo_error(e)
+        log_repl_error('Unexpected Error in ReplServer.check', err: e)
+      end
+
+      # Checks basics to see if we can run
+      # Logs errors (should be visible in whatever is running the server
+      # @return true can run when true or false (can't run)
+      def can_run?
+        @errs = []
+
+        # check if replication slot is setup
+        unless Koyo::Repl::Database.replication_slot_exists?
+          errs << "Error: Replication Slot doesn't exist. "\
+                   'See koyo-postgres-replication gem for how to set this up.'
+        end
+
+        # if there were any errors - let user know we're shutting down
+        errs << 'Shutting down' unless errs.empty?
+
+        errs.each { |msg| log_repl_error(msg) }
+
+        errs.empty?
+      end
+
+      # Finds all models that that implement 'self.koyo_repl_handler'
+      # This is only run once - during server spin up
+      def self.tables_that_handle_koyo_replication
+        return tables if tables.present?
+
+        log_repl_info('Init: Finding models that support '\
+                      "#{TABLE_METHOD_NAME}")
+        tables = {}
+        ActiveRecord::Base.connection.tables.map do |model|
+          klass_name = model.capitalize.singularize.camelize
+          klass = klass_name.constantize
+          next unless klass.methods.include?(TABLE_METHOD_NAME)
+
+          tables[klass.table_name] = klass_name
+        rescue NameError # filters out stuff like SchemaMigration
+          log_repl_info("Init: ignoring model #{klass_name}")
+        rescue StandardError => e
+          Koyo::Repl::EventHandlerService.koyo_error(e)
+          log_repl_error('Unexpected Error in '\
+              'ReplServer.tables_that_handle_koyo_replication',
+                         err: e)
+        end
+        tables.each do |t|
+          log_repl_info("Init: registering handler #{t}")
+        end
+        tables
+      end
+    end
+  end
+end

data/lib/koyo/repl/railtie.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'rails'
+
+module Koyo
+  module Repl
+    # Adds rake tasks accessible to Rails project
+    class Railtie < Rails::Railtie
+      railtie_name :koyo_repl
+
+      rake_tasks do
+        namespace :koyo do
+          namespace :repl do
+            desc 'Diagnostics: Basic setup/state information'
+            task diagnostics: :environment do
+              puts ''
+              puts '-' * 80
+              puts 'Koyo::Repl::Diagnostic'
+              puts Koyo::Repl::Diagnostics.new.rake_info.join("\n")
+              puts '-' * 80
+              puts ''
+            end
+
+            desc 'Install templates unless they already exist'
+            task install: :environment do
+              Koyo::Repl::Install.copy!
+            end
+
+            desc 'Process replication slot events - only run this server ONCE'
+            task run_server: :environment do
+              puts 'Running Koyo::Repl::PostgresServer.run!'
+              Koyo::Repl::PostgresServer.run!
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/koyo/repl/templates/koyo_postgres_replication_config.txt

@@ -0,0 +1,54 @@
+Koyo::Repl.configure do |config|
+  # Create the replication slot if it doesn't exist. Defaults to true
+  # You can set this with an ENV as well: 
+  #   KOYO_REPL_AUTO_CREATE_REPLICATION_SLOT
+  # config.auto_create_replication_slot = true
+
+  # Allows you to override the prefix used if you're using ENV to configure
+  # things. Defaults to KOYO_REPL
+  # config.config_prefix = 'KOYO_REPL'
+
+  # DB connection name in config/database.yml. Defaults to Rails.env
+  # (so standard connection on most rails app). We add this because you need
+  # admin priveleges to use replication and some companies have problems with
+  # this. Whatever this is called it will have Rails.env tacked on so if it's
+  # replication - the connection would be "replciation_#{Rails.env}"
+  # You can set this with an ENV as well: 
+  #   KOYO_REPL_DATABASE_NAME
+  # config.database_name = 'replication'
+
+  # Disable logging. Not recommended.
+  # You can set this with an ENV as well: 
+  #   KOYO_REPL_DISABLE_LOGGING
+  # config.disable_logging = true
+
+  # Determines the name of this replication slot. Defaults to
+  # koyo_repl_{Rails.env}.
+  # You can check replication slots that exist with:
+  #
+  #  select slot_name
+  #  from pg_replication_slots
+  #  where
+  #    and plugin = 'wal2json'
+  # You can set this with an ENV as well: 
+  #   KOYO_REPL_SLOT
+  # config.slot = "koyo_repl_#{Rails.env}"
+
+  # Time to wait before checking Replication Slot again in seconds
+  # Note: that if there 10,000 things on the replciation-queue it will
+  # process all of those as fast as possible, then pause for this many
+  # seconds before re-checking the replication-queue
+  # You can set this with an ENV as well: 
+  #   KOYO_REPL_SQL_DELAY
+  # config.sql_delay = 1
+
+  # When true we only "peek" the replication slot
+  # Peek (when this is false):
+  #   leaves the data on the postgres-replication queue
+  # Read (when this is true):
+  #   removes data from the postgres-replication queue
+  # Defaults to false
+  # You can set this with an ENV as well: 
+  #   KOYO_REPL_TEST_MODE
+  # config.test_mode = false
+end

data/lib/koyo/repl/templates/koyo_repl_handler_service.txt

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+# You can override these methods to do things like ping Slack, retart
+# crashed things, handle errors, handle logging, handle all replicaiton events 
+# (instead of doing this in models)
+class KoyoReplHandlerService < Koyo::Repl::EventHandlerService
+  class << self
+    # This is called for every create/update/delete action called on any table
+    # This can be used if you want to process all replication in one place
+    # instead of using the model level mixins. See README
+    # For an app with a lot of DB churn you want this to be as fast as possible
+    # You don't want do API calls from here, and shouldn't do DB updates from 
+    # here (but if you do, be super careful of infinite loops)
+    # def koyo_handle_all_replication(row)
+    #   case row.table
+    #   when 'users'
+    #     # do something with user like
+    #     # ElasticSearchUpdaterServer.perform_async(row.id)
+    #     # or grab the user record
+    #     # user = User.find(row.id)
+    #   end
+    #   super
+    # end
+    def koyo_handle_all_replication(row); end
+
+    # Called whenever an error is raised in Koyo::Repl code
+    # Examples would be to log to Slack, or restart something that's crashed
+    def koyo_error(err); end
+
+    # log_level: :debug, :info, :warn, :error, :fatal
+    # Example of message
+    # source=KoyoReplication logid=d7f1f0bb2a
+    #   message=Init: Finding models that support koyo_repl_handler
+    # You can use this as a catch all for any log event or use methods
+    # below if that's easier. You could use this for monitoring type activites
+    def koyo_log_event(message, log_level); end
+
+    # Called whenever Rails.logger.debug is called from Koyo::Repl code
+    def koyo_log_event_debug(message); end
+
+    # Called whenever Rails.logger.debug is called from Koyo::Repl code
+    def koyo_log_event_info(message); end
+
+    # Called whenever Rails.logger.debug is called from Koyo::Repl code
+    def koyo_log_event_warn(message); end
+
+    # Called whenever Rails.logger.debug is called from Koyo::Repl code
+    def koyo_log_event_error(message); end
+
+    # Called whenever Rails.logger.debug is called from Koyo::Repl code
+    def koyo_log_event_fatal(message); end
+  end
+end

data/lib/koyo/repl/templates/koyo_repl_model_example.txt

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# Basic model example
+class KoyoReplModelExample < ApplicationRecord
+  include Koyo::Repl::Mod
+
+  # register method for replication
+  koyo_repl_handler :handle_replication
+
+  # This is called when a row is created/updated/deleted
+  # You don't want to do DB updates from this or you will likely
+  # create an infinite loop
+  # This needs to be REALLY fast if you have a lot of db traffic
+  # For example: if you're doing something like calling an API from this
+  # method you should async it (put it in Sidekiq, ActiveJob, etc)
+  def self.handle_replication(row)
+    puts row.to_yaml
+  end
+end

data/lib/koyo/repl/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Koyo
+  module Repl
+    VERSION = '0.1.0.pre'
+  end
+end

data/lib/koyo.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Koyo
+  # Main replication namespace
+  module Repl
+    require_relative 'koyo/repl/configuration'
+    require_relative 'koyo/repl/data'
+    require_relative 'koyo/repl/database'
+    require_relative 'koyo/repl/data_row'
+    require_relative 'koyo/repl/diagnostics'
+    require_relative 'koyo/repl/event_handler_service'
+    require_relative 'koyo/repl/install'
+    require_relative 'koyo/repl/log'
+    require_relative 'koyo/repl/mod'
+    require_relative 'koyo/repl/postgres_server'
+    require_relative 'koyo/repl/railtie' # if defined?(Rails)
+    require_relative 'koyo/repl/version'
+
+    def self.config
+      @config ||= Configuration.new
+    end
+
+    def self.configure
+      yield(config)
+    end
+  end
+end

data/lib/koyo_postgres_replication.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+# Dummy mod for gemspec
+module KoyoPostgresReplication
+end