koyo-postgres-replication 0.1.0.pre

data/lib/koyo/repl/configuration.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Koyo
+  module Repl
+    # Supports config/initializer or ENV and adds defaults
+    # lib/koyo/repl/templates/koyo_postges_replication_config is
+    # copied to config/initializers when `rake koyo:repl:install` is
+    # run. If you set the associated ENV it will override these settings.
+    class Configuration
+      attr_writer :auto_create_replication_slot,
+                  :config_prefix,
+                  :database_name,
+                  :disable_logging,
+                  :slot,
+                  :sql_delay,
+                  :test_mode
+
+      # Try to auto create replication slot if it doesn't exist
+      # Defaults to true
+      # Override with ENV['KOYO_REPL_AUTO_CREATE_REPLICATION_SLOT']
+      def auto_create_replication_slot
+        val = @auto_create_replication_slot ||
+              ENV["#{config_prefix}_AUTO_CREATE_REPLICATION_SLOT"] ||
+              'true'
+        Koyo::Repl::Database.to_bool(val)
+      end
+
+      # Overrides the default prefix of ENV variables
+      # Override with ENV['KOYO_REPL_CONFIG_PREFIX']
+      def config_prefix
+        @config_prefix || ENV['KOYO_REPL_CONFIG_PREFIX'] || 'KOYO_REPL'
+      end
+
+      # Name of config/database.yml connection to use for replication
+      #
+      # Since this requires admin priveleges you might want to use
+      # a different connection to prevent all rails actions having
+      # admin priveleges to your DB. Default to whatever the default
+      # DB is for the project
+      # Override with ENV['KOYO_REPL_DB_CONN_NAME']
+      def database_name
+        @database_name || ENV["#{config_prefix}_DATABASE_NAME"]
+      end
+
+      # Disables logging (not recommended)
+      # Defaults to false
+      # Override with ENV['KOYO_REPL_DISABLE_LOGGING']
+      def disable_logging
+        Koyo::Repl::Database.to_bool(@disable_logging ||
+          ENV["#{config_prefix}_DISABLE_LOGGING"])
+      end
+
+      # Replication Slot name - can be any string - but must be
+      # unique to your database server.
+      #
+      # This is the name of the replication slot in postgres
+      # You can check replication slots that exist with:
+      #
+      #  select slot_name
+      #  from pg_replication_slots
+      #  where
+      #    and plugin = 'wal2json'
+      #
+      # Override with ENV['KOYO_REPL_SLOT']
+      def slot
+        @slot ||
+          ENV["#{config_prefix}_SLOT"] ||
+          "koyo_repl_#{Koyo::Repl::Database.current_db_name}_#{Rails.env}"
+      end
+
+      # Time to wait before checking Replication Slot again in seconds
+      # Note: that if there 10,000 things on the replciation-queue it will
+      # process all of those as fast as possible, then pause for this many
+      # seconds before re-checking the replication-queue
+      # Overide with ENV['KOYO_REPL_SQL_DELAY']
+      def sql_delay
+        @sql_delay || (ENV["#{config_prefix}_SQL_DELAY"] || 1).to_i
+      end
+
+      # When true we only "peek" the replication slot
+      # Peek (when this is false):
+      #   leaves the data on the postgres-replication queue
+      # Read (when this is true):
+      #   removes data from the postgres-replication queue
+      # Defaults to false
+      # Override with ENV['KOYO_REPL_TEST_MODE']
+      def test_mode
+        val = @test_mode || ENV["#{config_prefix}_TEST_MODE"]
+        Koyo::Repl::Database.to_bool(val)
+      end
+
+      # Helper method that converts config settings into a hash
+      def to_h
+        {
+          auto_create_replication_slot: auto_create_replication_slot,
+          config_prefix: config_prefix,
+          database_name: database_name,
+          slot: slot,
+          sql_delay: sql_delay,
+          test_mode: test_mode
+        }
+      end
+
+      # Helper method that converts config settings into a string
+      def to_s
+        to_h.map { |k, v| "#{k}: #{v}" }.join("\n")
+      end
+    end
+  end
+end

data/lib/koyo/repl/data.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Koyo
+  module Repl
+    # Class wrapper for replication data pulls
+    #   Example data packet
+    #
+    #   {
+    #   "change": [
+    #     {
+    #       "kind": "update",
+    #       "schema": "public",
+    #       "table": "users",
+    #       "columnnames": ["id", "email", ...],
+    #       "columntypes": ["integer", "character varying(255)", ...],
+    #       "columnvalues": [120665, "", ...]
+    #       "oldkeys": {
+    #         "keynames": ["id"],
+    #         "keytypes": ["integer"],
+    #         "keyvalues": [120665]
+    #       }
+    #     }
+    #   ]
+    # }
+    class Data
+      attr_accessor :row, # raw results from db
+                    :lsn, # ???
+                    :xid  # uniq id of row returned
+
+      # Takes a row from ReplDatabase.(peek_slot/read_slot!)
+      # @see For details on `row` see:
+      # https://github.com/wiseleyb/koyo-postgres-replication/wiki/Koyo::Repl::DataRow-data-spec
+      def initialize(row)
+        @row = row
+        @lsn = row['lsn']
+        @xid = row['xid']
+        # TODO: find faster JSON lib for this
+        @data_rows = Koyo::Repl::Database.parse_json(row['data'])['change']
+      end
+
+      # Collection of Koyo::Repl::DataRow
+      #
+      # When you read from a replicaiton slot it can return 0-n change
+      # events. In general this system works on a row based level,
+      # which is a bit slower but is simpler to code.
+      def rows
+        # @data_rows.map { |d| ReplDataRow.new(xid, d) }
+        @data_rows.map { |d| Koyo::Repl::DataRow.new(d) }
+      end
+    end
+  end
+end

data/lib/koyo/repl/data_row.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Koyo
+  module Repl
+    # Class wrapper for replication row row
+    # @see
+    # https://github.com/wiseleyb/koyo-postgres-replication/wiki/Koyo::Repl::DataRow-row-spec
+    # @see For details on row
+    class DataRow
+      attr_accessor :row,           # raw json of row - see example below
+                    :kind,          # insert/update/delete
+                    :schema,        # always public for this - not needed
+                    :table,         # table being changed
+                    :id,            # table.id
+                    :id_type,       # integer/uuid
+                    :columns,       # all columns from table - array
+                    :column_types,  # all types of columns - array
+                    :values         # all values from table - array
+
+      # Initialized attributes
+      def initialize(row)
+        @row = row
+        @kind = @row['kind']
+        @schema = @row['schema']
+        @table = @row['table']
+        @columns = @row['columnnames']
+        @column_types = @row['columntypes']
+        @values = @row['columnvalues']
+        check_set_primary_keys
+      end
+
+      # This doesn't work for multiple primary keys right now
+      #
+      # WARN: this breaks for multiple primary keys
+      def check_set_primary_keys
+        if @row['oldkeys']
+          if @row['oldkeys']['keynames'].size > 1
+            raise "This doesn't support multiple keys right now"
+          end
+
+          @id = @row['oldkeys']['keyvalues'].first
+          @id_type = @row['oldkeys']['keytypes'].first
+        else
+          @id = val(:id)
+          @id_type = type(:id)
+        end
+      end
+
+      # Gets a value for a name from columnsvalues
+      # @param name column name
+      def val(name)
+        values[columns.index(name.to_s)]
+      end
+
+      # Get a val type from columntypes
+      # @param name column name
+      def type(name)
+        column_types[columns.index(name.to_s)]
+      end
+    end
+  end
+end
+
+#   Example row packet
+#
+#   {
+#   "change": [
+#     {
+#       "kind": "delete",
+#       "schema": "public",
+#       "table": "users",
+#       "oldkeys": {
+#         "keynames": ["id"],
+#         "keytypes": ["integer"],
+#         "keyvalues": [123]
+#       }
+#     },
+#     {
+#       "kind": "insert",
+#       "schema": "public",
+#       "table": "users",
+#       "columnnames": ["id", "name", "email", ...],
+#       "columntypes": ["integer", "character varying", ...],
+#       "columnvalues": [234, "User", ...]
+#     },
+#     {
+#       "kind": "update",
+#       "schema": "public",
+#       "table": "users",
+#       "columnnames": ["id", "name", "email", ...],
+#       "columntypes": ["integer", "character varying", ...],
+#       "columnvalues": [234, "User", ...]
+#       "oldkeys": {
+#         "keynames": ["id"],
+#         "keytypes": ["integer"],
+#         "keyvalues": [233]
+#       }
+#     }
+#   ]
+# }

data/lib/koyo/repl/database.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module Koyo
+  module Repl
+    # Basic utilities for postgres replication
+    # @see Postgres doc for more options
+    # https://www.postgresql.org/docs/9.4/logicaldecoding-example.html
+    class Database
+      class << self
+        # TODO: refactor this out of the config class
+        # DB connection name in config/database.yml. Defaults to Rails.env
+        # (so standard connection on most rails app). We add this because you need
+        # admin priveleges to use replication and some companies have problems with
+        # this. Whatever this is called it will have Rails.env tacked on so if it's
+        # replication - the connection would be "replciation_#{Rails.env}"
+        def conn
+          return @conn if @conn
+
+          conn_name = Koyo::Repl.config.database_name
+
+          unless conn_name
+            @conn = ActiveRecord::Base.connection
+            return @conn
+          end
+
+          conn_name = "#{conn_name}_#{Rails.env}"
+
+          msg = "source=KoyoReplication Connecting to #{conn_name}"
+          Rails.logger.info msg
+
+          config =
+            ApplicationRecord.configurations.find_db_config(conn_name)
+          ActiveRecord::Base.establish_connection config
+          @conn = ActiveRecord::Base.connection
+          @conn
+        end
+
+        # Reads from the replication slot.
+        # Reading from this marks the rows read (so you won't see them again)
+        # For testing you can use `peek_slot` if you want to - which will keep
+        # the data in the slot. A known issue is that this slot can grow so
+        # large that you it'll time out when trying to read from it. This is a
+        # major downside of this approach. Please open an issue if you know a
+        # solution.
+        def read_slot!
+          sql = %(
+          SELECT *
+          FROM pg_logical_slot_get_changes('#{config_slot}',
+                                           NULL,
+                                           NULL,
+                                           'pretty-print',
+                                           '1');
+        )
+          exec_sql(sql)
+        end
+
+        # Peeks at data in the replication-slot. Use this for debugging. This
+        # will leave the data in the replication-slot. If
+        # Configuration.test_mode=true the code will default to peek'ing
+        # instead of reading.
+        def peek_slot
+          sql = %(
+          SELECT *
+          FROM pg_logical_slot_peek_changes('#{config_slot}',
+                                            NULL,
+                                            NULL,
+                                            'pretty-print',
+                                            '1');
+        )
+          exec_sql(sql)
+        end
+
+        # Checks count of current slot
+        def replication_slot_count
+          sql = %(
+          SELECT count(*)
+          FROM pg_logical_slot_peek_changes('#{config_slot}',
+                                            NULL,
+                                            NULL,
+                                            'pretty-print',
+                                            '1');
+        )
+          exec_sql(sql).first['count'].to_i
+        end
+
+        # Checks to see if the replication slot exists
+        def replication_slot_exists?
+          sql = %(
+          select count(*)
+          from pg_replication_slots
+          where
+            slot_name = '#{config_slot}'
+            and database = '#{current_db_name}'
+            and plugin = 'wal2json'
+        )
+          exec_sql(sql).first['count'].to_i.positive?
+        end
+
+        # Returns all data for current replication slot.
+        def replication_slot
+          sql = %(
+          select *
+          from pg_replication_slots
+          where plugin = 'wal2json'
+          and slot_name = '#{config_slot}'
+          and database = '#{current_db_name}'
+        )
+          exec_sql(sql).first
+        end
+
+        # Returns all replication slots on the system that support wal2json
+        def replication_slots
+          sql = %(
+          select *
+          from pg_replication_slots
+          where plugin = 'wal2json'
+        )
+          exec_sql(sql)
+        end
+
+        # Creates a replication slot. You need admin priveleges for this.
+        def create_replication_slot!
+          return if replication_slot_exists?
+
+          sql = %(
+          SELECT 'init'
+          FROM pg_create_logical_replication_slot('#{config_slot}',
+                                                  'wal2json')
+        )
+          exec_sql(sql)
+        end
+
+        # Deletes replication slot. You need admin priveleges for this.
+        def delete_replication_slot!
+          return unless replication_slot_exists?
+
+          sql = %(select pg_drop_replication_slot('#{config_slot}'))
+          exec_sql(sql)
+        end
+
+        # Drops and recreates replication slot
+        def drop_create_slot!
+          Koyo::Repl::Database.delete_replication_slot!
+          Koyo::Repl::Database.create_replication_slot!
+        end
+
+        # Checks the wal_level - which should be "logical" if things are setup
+        # properly. You can change the wal_level in postgres config. See the
+        # README for details on on this. When you change this you need to
+        # restart the postgres server
+        def wal_level
+          sql = %(show wal_level)
+          exec_sql(sql).first['wal_level']
+        end
+
+        # Helper - just returns the configured database name being used. Can
+        # be changed using Configuration.slot
+        def current_db_name
+          Rails.configuration.database_configuration[Rails.env]['database']
+        end
+
+        # Runs SQL commands
+        def exec_sql(sql)
+          # ActiveRecord::Base.connection.execute(sql)
+          conn.execute(sql)
+        end
+
+        # wrap this to support faster JSON parsing in the future
+        def parse_json(json)
+          JSON.parse(json)
+        end
+
+        def config_slot
+          Koyo::Repl.config.slot
+        end
+
+        def to_bool(val)
+          %w[1 true t yes].include?(val.to_s.downcase.strip)
+        end
+      end
+    end
+  end
+end

data/lib/koyo/repl/diagnostics.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Koyo
+  module Repl
+    # Provides state and debugging info for Repl setup
+    # can be run with rake koyo::repl::diagnostics
+    class Diagnostics
+      # For use with rake koyo::repl::diagnostics
+      # Outputs repl setup and current state info
+      def rake_info
+        [
+          "Config settings: \n#{h_to_s(Koyo::Repl.config.to_h)}",
+          "Replication slot exists: #{replication_slot_exists?}",
+          "Registered tables: \n#{h_to_s(registered_tables)}",
+          "Can connect to db: #{can_connect?}",
+          "Connection adapter: #{adapter_name}",
+          "Wal Level (should be 'logical'): #{wal_level}",
+          "Can access replication slot: #{can_access_replication_slot?}",
+          "Replication slot count: #{repl_count}"
+        ]
+      end
+
+      # Checks if replication slot exists.
+      # TODO: maybe this should create if it doesn't?
+      def replication_slot_exists?
+        Koyo::Repl::Database.replication_slot_exists?
+      rescue StandardError => e
+        "Error: #{e.message}"
+      end
+
+      # Returns list of models that have registered a call back
+      def registered_tables
+        res =
+          Koyo::Repl::PostgresServer.tables_that_handle_koyo_replication || {}
+        if res == {}
+          res = {
+            warning: 'No tables registered - see example file in '\
+                     'app/models/koyo_repl_model_example.rb to see '\
+                     'how to monitor replication for a table from a model. '\
+                     'This is optional - you can just use '\
+                     'app/models/koyo_repl_handler_service.rb as a catch all '\
+                     'for all replication events if you want.'
+          }
+        end
+        res
+      rescue StandardError => e
+        "Error: #{e.message}"
+      end
+
+      # Checks connection to database
+      def can_connect?
+        Koyo::Repl::Database.conn.execute('select now()')
+        true
+      rescue StandardError => e
+        "Error: #{e.message}"
+      end
+
+      # Checks access/permissions to replication slot
+      def can_access_replication_slot?
+        Koyo::Repl::Database.peek_slot
+        true
+      rescue StandardError => e
+        "Error: #{e.message}"
+      end
+
+      # Checks that replication slot exists
+      def repl_count
+        Koyo::Repl::Database.replication_slot_count
+      rescue StandardError => e
+        "Error: #{e.message}"
+      end
+
+      # Returns configured database name
+      def adapter_name
+        Koyo::Repl::Datasbase.conn.adapter_name
+      rescue StandardError => e
+        "Error: #{e.message}"
+      end
+
+      # Returns configured wal_level. Should be "logical"
+      def wal_level
+        Koyo::Repl::Database.wal_level
+      rescue StandardError => e
+        "Error: #{e.message}"
+      end
+
+      # Helper - outputs a hash
+      def h_to_s(hash)
+        hash.map { |k, v| "  #{k}: #{v}" }.join("\n")
+      end
+    end
+  end
+end

data/lib/koyo/repl/event_handler_service.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Koyo
+  module Repl
+    # Calls back to Rails project
+    # This is primarily for overriding in the rails app to do things like ping
+    # Slack, or restart things on errors, etc
+    class EventHandlerService
+      class << self
+        # example row:
+        # User.handle_replcation called
+        def koyo_handle_all_replication(row)
+          KoyoReplHandlerService.koyo_handle_all_replication(row)
+        end
+
+        # Called whenever an error is raised in Koyo::Repl code
+        def koyo_error(err)
+          KoyoReplHandlerService.koyo_error(err)
+        end
+
+        # log_level: :debug, :info, :warn, :error, :fatal
+        # Example of message
+        # source=KoyoReplication logid=d7f1f0bb2a
+        #   message=Init: Finding models that support koyo_repl_handler
+        # You can use this as a catch all for any log event or use methods
+        # below if that's easier
+        def koyo_log_event(message, log_level)
+          KoyoReplHandlerService.koyo_log_event(message, log_level)
+        end
+
+        # Called whenever Rails.logger.debug is called from Koyo::Repl code
+        def koyo_log_event_debug(message)
+          KoyoReplHandlerService.koyo_log_event_debug(message)
+        end
+
+        # Called whenever Rails.logger.debug is called from Koyo::Repl code
+        def koyo_log_event_info(message)
+          KoyoReplHandlerService.koyo_log_event_info(message)
+        end
+
+        # Called whenever Rails.logger.debug is called from Koyo::Repl code
+        def koyo_log_event_warn(message)
+          KoyoReplHandlerService.koyo_log_event_warn(message)
+        end
+
+        # Called whenever Rails.logger.debug is called from Koyo::Repl code
+        def koyo_log_event_error(message)
+          KoyoReplHandlerService.koyo_log_event_error(message)
+        end
+
+        # Called whenever Rails.logger.debug is called from Koyo::Repl code
+        def koyo_log_event_fatal(message)
+          KoyoReplHandlerService.koyo_log_event_fatal(message)
+        end
+      end
+    end
+  end
+end

data/lib/koyo/repl/install.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Koyo
+  module Repl
+    # Copies required files to Rails project
+    class Install
+      def self.copy!
+        kri = Koyo::Repl::Install.new
+        kri.copy!
+        Koyo::Repl::Database.drop_create_slot!
+      end
+
+      # Copies files unless they already exist
+      def copy!
+        debugp ''
+        debugp '-' * 80
+        copy("#{template_path}/koyo_postgres_replication_config.txt",
+             "#{rails_path}/config/initializers/"\
+               'koyo_postgres_replication_config.rb')
+
+        copy("#{template_path}/koyo_repl_handler_service.txt",
+             "#{rails_path}/app/models/koyo_repl_handler_service.rb")
+
+        copy("#{template_path}/koyo_repl_model_example.txt",
+             "#{rails_path}/app/models/koyo_repl_model_example.rb")
+        debugp '-' * 80
+      end
+
+      # Debugging helper
+      def debugp(msg)
+        puts msg unless Rails.env.test? # don't pollute spec output
+      end
+
+      # Helper for checking if files exist
+      # @param fname file name (with path)
+      def file_exists?(fname)
+        if File.exist?(fname)
+          puts "SKIPPING: #{fname} exists. Delete this file to recreated it."
+          return true
+        end
+        false
+      end
+
+      # Copies individual file
+      # @param from_fname file to copy from
+      # @param to_fname file to copy to
+      def copy(from_fname, to_fname)
+        return if file_exists?(to_fname)
+
+        puts "ADDING #{to_fname}"
+        dir_name = File.dirname(to_fname)
+        FileUtils.mkdir_p(dir_name)
+        FileUtils.cp(from_fname, to_fname.gsub('.txt', '.rb'))
+      end
+
+      # Path of template files to copy
+      def template_path
+        "#{File.dirname(__FILE__)}/templates"
+      end
+
+      # Rails path helper
+      def rails_path
+        Rails.root.to_s
+      end
+    end
+  end
+end