active_record_proxy_adapters 0.1.0

data/docker-compose.yml

@@ -0,0 +1,77 @@
+name: active_record_proxy_adapters
+
+x-postgres-common: &postgres-common
+  restart: always
+  user: postgres
+  healthcheck:
+    test: 'pg_isready -U postgres_primary_test --dbname=postgres'
+    interval: 10s
+    timeout: 5s
+    retries: 5
+  networks:
+    - postgres
+
+services:
+  app:
+    build:
+      args:
+        - RUBY_VERSION=${RUBY_VERSION:-3.2.3}
+        - RAILS_VERSION=${RAILS_VERSION:-~> 6.1.0}
+    container_name: app
+    image: active_record_proxy_adapters-app:${ENV_TAG:-latest}
+    tty: true
+    stdin_open: true
+    environment:
+      PGHOST: postgres_primary
+      PG_PRIMARY_USER: postgres_primary_test
+      PG_PRIMARY_PASSWORD: postgres_primary_test
+      PG_PRIMARY_HOST: postgres_primary
+      PG_PRIMARY_PORT: 5432
+      PG_REPLICA_USER: postgres_primary_test
+      PG_REPLICA_PASSWORD: postgres_primary_test
+      PG_REPLICA_HOST: postgres_replica
+      PG_REPLICA_PORT: 5432
+    depends_on:
+      - postgres_primary
+      - postgres_replica
+    networks:
+      - app
+      - postgres
+    volumes:
+      - .:/app
+  postgres_primary:
+    <<: *postgres-common
+    build:
+      context: .
+      dockerfile: postgres_primary.dockerfile
+      args:
+        - POSTGRES_LOGGING_COLLECTOR=${POSTGRES_LOGGING_COLLECTOR:-}
+        - POSTGRES_LOG_DESTINATION=${POSTGRES_LOG_DESTINATION:-}
+        - POSTGRES_LOG_STATEMENT=${POSTGRES_LOG_STATEMENT:-}
+        - REPLICA_USER=replicator
+        - REPLICA_PASSWORD=replicator
+    environment:
+      POSTGRES_DB: postgres
+      POSTGRES_USER: postgres_primary_test
+      POSTGRES_PASSWORD: postgres_primary_test
+      POSTGRES_HOST_AUTH_METHOD: "scram-sha-256\nhost replication all 0.0.0.0/0 md5"
+      POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256"
+
+  postgres_replica:
+    <<: *postgres-common
+    build:
+      context: .
+      dockerfile: postgres_replica.dockerfile
+    container_name: postgres_replica
+    environment:
+      PGUSER: replicator
+      PGPASSWORD: replicator
+      PRIMARY_DATABASE_HOST: postgres_primary
+    depends_on:
+      - postgres_primary
+networks:
+  app:
+  postgres:
+
+volumes:
+  postgres_primary:

data/lib/active_record/connection_adapters/postgresql_proxy_adapter.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "active_record/tasks/postgresql_proxy_database_tasks"
+require "active_record/connection_adapters/postgresql_adapter"
+require "active_record_proxy_adapters/active_record_context"
+require "active_record_proxy_adapters/hijackable"
+require "active_record_proxy_adapters/postgresql_proxy"
+
+module ActiveRecord
+  module ConnectionAdapters
+    # This adapter is a proxy to the original PostgreSQLAdapter, allowing the use of the
+    # ActiveRecordProxyAdapters::PrimaryReplicaProxy.
+    class PostgreSQLProxyAdapter < PostgreSQLAdapter
+      include ActiveRecordProxyAdapters::Hijackable
+
+      ADAPTER_NAME = "PostgreSQLProxy"
+
+      delegate_to_proxy :execute, :exec_query
+
+      unless ActiveRecordProxyAdapters::ActiveRecordContext.active_record_v8_0_or_greater?
+        delegate_to_proxy :exec_no_cache, :exec_cache
+      end
+
+      def initialize(...)
+        @proxy = ActiveRecordProxyAdapters::PostgreSQLProxy.new(self)
+
+        super
+      end
+
+      private
+
+      attr_reader :proxy
+    end
+  end
+end
+
+if ActiveRecordProxyAdapters::ActiveRecordContext.active_record_v7_2_or_greater?
+  ActiveRecord::ConnectionAdapters.register(
+    "postgresql_proxy",
+    "ActiveRecord::ConnectionAdapters::PostgreSQLProxyAdapter",
+    "active_record/connection_adapters/postgresql_proxy_adapter"
+  )
+end

data/lib/active_record/tasks/postgresql_proxy_database_tasks.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Tasks
+    # Defines the postgresql tasks for dropping, creating, loading schema and dumping schema.
+    # Bypasses all the proxy logic to send all requests to primary.
+    class PostgreSQLProxyDatabaseTasks < PostgreSQLDatabaseTasks
+      def create(...)
+        sticking_to_primary { super }
+      end
+
+      def drop(...)
+        sticking_to_primary { super }
+      end
+
+      def structure_dump(...)
+        sticking_to_primary { super }
+      end
+
+      def structure_load(...)
+        sticking_to_primary { super }
+      end
+
+      def purge(...)
+        sticking_to_primary { super }
+      end
+
+      private
+
+      def sticking_to_primary(&)
+        ActiveRecord::Base.connected_to(role: context.writing_role, &)
+      end
+
+      def context
+        ActiveRecordProxyAdapters::ActiveRecordContext.new
+      end
+    end
+  end
+end
+
+# Allow proxy adapter to run rake tasks, i.e. db:drop, db:create, db:schema:load db:migrate, etc...
+ActiveRecord::Tasks::DatabaseTasks.register_task(
+  /postgresql_proxy/,
+  "ActiveRecord::Tasks::PostgreSQLProxyDatabaseTasks"
+)

data/lib/active_record_proxy_adapters/active_record_context.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module ActiveRecordProxyAdapters
+  # Collection of helpers to handle common active record methods that are defined in different places in different
+  # versions of rails.
+  class ActiveRecordContext
+    delegate :reading_role, :reading_role=, :writing_role, :writing_role=, to: :ActiveRecord
+    delegate :legacy_connection_handling, :legacy_connection_handling=, to: :connection_handling_context
+    delegate :version, to: :ActiveRecord, prefix: :active_record
+
+    class << self
+      delegate_missing_to :new
+    end
+
+    NullConnectionHandlingContext = Class.new do
+      def legacy_connection_handling
+        false
+      end
+
+      def legacy_connection_handling=(_value)
+        nil
+      end
+    end
+
+    def connection_class_for(connection)
+      connection.connection_class || ActiveRecord::Base
+    end
+
+    def connection_handling_context
+      # This config option has been removed in Rails 7.1+
+      return NullConnectionHandlingContext.new if active_record_v7_1_or_greater?
+
+      ActiveRecord
+    end
+
+    def active_record_v7_1_or_greater?
+      active_record_version >= Gem::Version.new("7.1")
+    end
+
+    def active_record_v7_2_or_greater?
+      active_record_version >= Gem::Version.new("7.2")
+    end
+
+    def active_record_v8_0_or_greater?
+      active_record_version >= Gem::Version.new("8.0")
+    end
+  end
+end

data/lib/active_record_proxy_adapters/configuration.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/integer/time"
+
+module ActiveRecordProxyAdapters
+  # Provides a global configuration object to configure how the proxy should behave.
+  class Configuration
+    PROXY_DELAY      = 2.seconds.freeze
+    CHECKOUT_TIMEOUT = 2.seconds.freeze
+
+    # @return [ActiveSupport::Duration] How long the proxy should reroute all read requests to the primary database
+    #   since the latest write. Defaults to PROXY_DELAY.
+    attr_accessor :proxy_delay
+    # @return [ActiveSupport::Duration] How long the proxy should wait for a connection from the replica pool.
+    #   Defaults to CHECKOUT_TIMEOUT.
+    attr_accessor :checkout_timeout
+
+    def initialize
+      self.proxy_delay      = PROXY_DELAY
+      self.checkout_timeout = CHECKOUT_TIMEOUT
+    end
+  end
+end

data/lib/active_record_proxy_adapters/connection_handling.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "active_record/connection_adapters/postgresql_proxy_adapter"
+
+module ActiveRecordProxyAdapters
+  # Module to extend ActiveRecord::Base with the connection handling methods.
+  # Required to make adapter work in ActiveRecord versions <= 7.2.x
+  module ConnectionHandling
+    def postgresql_proxy_adapter_class
+      ::ActiveRecord::ConnectionAdapters::PostgreSQLProxyAdapter
+    end
+
+    # This method is a copy and paste from Rails' postgresql_connection,
+    # replacing PostgreSQLAdapter by PostgreSQLProxyAdapter
+    # This is required by ActiveRecord versions <= 7.2.x to establish a connection using the adapter.
+    def postgresql_proxy_connection(config) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      conn_params = config.symbolize_keys.compact
+
+      # Map ActiveRecords param names to PGs.
+      conn_params[:user]   = conn_params.delete(:username) if conn_params[:username]
+      conn_params[:dbname] = conn_params.delete(:database) if conn_params[:database]
+
+      # Forward only valid config params to PG::Connection.connect.
+      valid_conn_param_keys = PG::Connection.conndefaults_hash.keys + [:requiressl]
+      conn_params.slice!(*valid_conn_param_keys)
+
+      postgresql_proxy_adapter_class.new(
+        postgresql_proxy_adapter_class.new_client(conn_params),
+        logger,
+        conn_params,
+        config
+      )
+    end
+  end
+end

data/lib/active_record_proxy_adapters/hijackable.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "active_record/tasks/postgresql_proxy_database_tasks"
+require "active_record/connection_adapters/postgresql_adapter"
+require "active_record_proxy_adapters/primary_replica_proxy"
+
+module ActiveRecordProxyAdapters
+  # Defines mixins to delegate specific methods from the proxy to the adapter.
+  module Hijackable
+    extend ActiveSupport::Concern
+
+    class_methods do
+      # Renames the methods from the original Adapter using the proxy suffix (_unproxied)
+      # and delegate the original method name to the proxy.
+      # Example: delegate_to_proxy(:execute) creates a method `execute_unproxied`,
+      # while delegating :execute to the proxy.
+      # @param method_name [Array<Symbol>] the names of methods to be redefined.
+      def delegate_to_proxy(*method_names)
+        method_names.each do |method_name|
+          proxy_method_name = proxy_method_name_for(method_name)
+          proxy_method_private = private_method_defined?(method_name)
+
+          # some adapter methods are private. We need to make them public before aliasing.
+          public method_name if proxy_method_private
+
+          alias_method proxy_method_name, method_name
+
+          # If adapter method was originally private. We now make them private again.
+          private method_name, proxy_method_name if proxy_method_private
+        end
+
+        delegate(*method_names, to: :proxy)
+      end
+
+      private
+
+      def proxy_method_name_for(method_name)
+        :"#{method_name}#{ActiveRecordProxyAdapters::PrimaryReplicaProxy::UNPROXIED_METHOD_SUFFIX}"
+      end
+    end
+  end
+end

data/lib/active_record_proxy_adapters/postgresql_proxy.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "active_record_proxy_adapters/primary_replica_proxy"
+require "active_record_proxy_adapters/active_record_context"
+
+module ActiveRecordProxyAdapters
+  # Proxy to the original PostgreSQLAdapter, allowing the use of the ActiveRecordProxyAdapters::PrimaryReplicaProxy.
+  class PostgreSQLProxy < PrimaryReplicaProxy
+    # ActiveRecord::PostgreSQLAdapter methods that should be proxied.
+    hijack_method :execute, :exec_query
+
+    hijack_method :exec_no_cache, :exec_cache unless ActiveRecordContext.active_record_v8_0_or_greater?
+  end
+end

data/lib/active_record_proxy_adapters/primary_replica_proxy.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require "active_record_proxy_adapters/configuration"
+require "active_support/core_ext/module/delegation"
+require "active_support/core_ext/object/blank"
+require "concurrent-ruby"
+require "active_record_proxy_adapters/active_record_context"
+
+module ActiveRecordProxyAdapters
+  # This is the base class for all proxies. It defines the methods that should be proxied
+  # and the logic to determine which database to use.
+  class PrimaryReplicaProxy # rubocop:disable Metrics/ClassLength
+    # All queries that match these patterns should be sent to the primary database
+    SQL_PRIMARY_MATCHERS = [
+      /\A\s*select.+for update\Z/i, /select.+lock in share mode\Z/i,
+      /\A\s*select.+(nextval|currval|lastval|get_lock|release_lock|pg_advisory_lock|pg_advisory_unlock)\(/i
+    ].map(&:freeze).freeze
+    # All queries that match these patterns should be sent to the replica database
+    SQL_REPLICA_MATCHERS     = [/\A\s*(select|with.+\)\s*select)\s/i].map(&:freeze).freeze
+    # All queries that match these patterns should be sent to all databases
+    SQL_ALL_MATCHERS         = [/\A\s*set\s/i].map(&:freeze).freeze
+    # Local sets queries should not be sent to all datbases
+    SQL_SKIP_ALL_MATCHERS    = [/\A\s*set\s+local\s/i].map(&:freeze).freeze
+    # These patterns define which database statments are considered write statments, so we can shortly re-route all
+    # requests to the primary database so the replica has time to replicate
+    WRITE_STATEMENT_MATCHERS = [/\ABEGIN/i, /\ACOMMIT/i, /INSERT\sINTO\s/i, /UPDATE\s/i, /DELETE\sFROM\s/i,
+                                /DROP\s/i].map(&:freeze).freeze
+    UNPROXIED_METHOD_SUFFIX  = "_unproxied"
+
+    # Defines which methods should be hijacked from the original adapter and use the proxy
+    # @param method_names [Array<Symbol>] the list of method names from the adapter
+    def self.hijack_method(*method_names) # rubocop:disable Metrics/MethodLength
+      @hijacked_methods ||= Set.new
+      @hijacked_methods += Set.new(method_names)
+
+      method_names.each do |method_name|
+        define_method(method_name) do |*args, **kwargs, &block|
+          proxy_bypass_method = "#{method_name}#{UNPROXIED_METHOD_SUFFIX}"
+          sql_string          = coerce_query_to_string(args.first)
+
+          appropriate_connection(sql_string) do |conn|
+            method_to_call = conn == primary_connection ? proxy_bypass_method : method_name
+
+            conn.send(method_to_call, *args, **kwargs, &block)
+          end
+        end
+      end
+    end
+
+    def self.hijacked_methods
+      @hijacked_methods.to_a
+    end
+
+    # @param primary_connection [ActiveRecord::ConnectionAdatpers::AbstractAdapter]
+    def initialize(primary_connection)
+      @primary_connection    = primary_connection
+      @last_write_at         = 0
+      @active_record_context = ActiveRecordContext.new
+    end
+
+    private
+
+    attr_reader :primary_connection, :last_write_at, :active_record_context
+
+    delegate :connected_to_stack, to: :connection_class
+    delegate :reading_role, :writing_role, to: :active_record_context
+
+    def connection_class
+      active_record_context.connection_class_for(primary_connection)
+    end
+
+    def replica_pool
+      ActiveRecord::Base.connected_to(role: reading_role) { ActiveRecord::Base.connection_pool }
+    end
+
+    def coerce_query_to_string(sql_or_arel)
+      sql_or_arel.respond_to?(:to_sql) ? sql_or_arel.to_sql : sql_or_arel.to_s
+    end
+
+    def appropriate_connection(sql_string, &block)
+      roles_for(sql_string).map do |role|
+        connection_for(role, sql_string) do |connection|
+          block.call(connection)
+        end
+      end.last
+    end
+
+    def roles_for(sql_string)
+      return [top_of_connection_stack_role] if top_of_connection_stack_role.present?
+
+      if need_all?(sql_string)
+        [reading_role, writing_role]
+      elsif need_primary?(sql_string)
+        [writing_role]
+      else
+        [reading_role]
+      end
+    end
+
+    def top_of_connection_stack_role
+      return if connected_to_stack.empty?
+
+      top = connected_to_stack.last
+      role = top[:role]
+      return unless role.present?
+
+      [reading_role, writing_role].include?(role) ? role : nil
+    end
+
+    def connection_for(role, sql_string) # rubocop:disable Metrics/MethodLength
+      connection = if role == writing_role
+                     primary_connection
+                   else
+                     begin
+                       replica_pool.checkout(checkout_timeout)
+                     # rescue NoDatabaseError to avoid crashing when running db:create rake task
+                     # rescue ConnectionNotEstablished to handle connectivity issues in the replica
+                     # (for example, replication delay)
+                     rescue ActiveRecord::NoDatabaseError, ActiveRecord::ConnectionNotEstablished
+                       primary_connection
+                     end
+                   end
+
+      result = yield(connection)
+      update_primary_latest_write_timestamp if !replica_connection?(connection) && write_statement?(sql_string)
+
+      result
+    ensure
+      replica_connection?(connection) && replica_pool.checkin(connection)
+    end
+
+    def replica_connection?(connection)
+      connection != primary_connection
+    end
+
+    # @return [TrueClass] if there has been a write within the last {#proxy_delay} seconds
+    # @return [TrueClass] if sql_string matches a write statement (i.e. INSERT, UPDATE, DELETE, SELECT FOR UPDATE)
+    # @return [FalseClass] if sql_string matches a read statement (i.e. SELECT)
+    def need_primary?(sql_string)
+      return true if recent_write_to_primary?
+
+      return true  if in_transaction?
+      return true  if SQL_PRIMARY_MATCHERS.any?(&match_sql?(sql_string))
+      return false if SQL_REPLICA_MATCHERS.any?(&match_sql?(sql_string))
+
+      true
+    end
+
+    def need_all?(sql_string)
+      return false if SQL_SKIP_ALL_MATCHERS.any?(&match_sql?(sql_string))
+
+      SQL_ALL_MATCHERS.any?(&match_sql?(sql_string))
+    end
+
+    def write_statement?(sql_string)
+      WRITE_STATEMENT_MATCHERS.any?(&match_sql?(sql_string))
+    end
+
+    def match_sql?(sql_string)
+      proc { |matcher| matcher.match?(sql_string) }
+    end
+
+    # TODO: implement a context API to re-route requests to the primary database if a recent query was sent to it to
+    # avoid replication delay issues
+    # @return Boolean
+    def recent_write_to_primary?
+      Concurrent.monotonic_time - last_write_at < proxy_delay
+    end
+
+    def in_transaction?
+      primary_connection.open_transactions.positive?
+    end
+
+    def update_primary_latest_write_timestamp
+      @last_write_at = Concurrent.monotonic_time
+    end
+
+    def proxy_delay
+      proxy_config.proxy_delay
+    end
+
+    def checkout_timeout
+      proxy_config.checkout_timeout
+    end
+
+    def proxy_config
+      ActiveRecordProxyAdapters.config
+    end
+  end
+end

data/lib/active_record_proxy_adapters/railtie.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "active_support"
+
+module ActiveRecordProxyAdapters
+  # Hooks into rails boot process to extend ActiveRecord with the proxy adapter.
+  class Railtie < Rails::Railtie
+    ActiveSupport.on_load :active_record do
+      require "active_record_proxy_adapters/connection_handling"
+      ActiveRecord::Base.extend(ActiveRecordProxyAdapters::ConnectionHandling)
+    end
+
+    config.to_prepare do
+      Rails.autoloaders.each do |autoloader|
+        autoloader.inflector.inflect(
+          "postgresql_proxy_adapter" => "PostgreSQLProxyAdapter"
+        )
+      end
+    end
+  end
+end

data/lib/active_record_proxy_adapters/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ActiveRecordProxyAdapters
+  VERSION = "0.1.0"
+end

data/lib/active_record_proxy_adapters.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "active_record_proxy_adapters/version"
+require "active_record_proxy_adapters/configuration"
+
+# The gem namespace.
+module ActiveRecordProxyAdapters
+  class Error < StandardError; end
+
+  module_function
+
+  def configure
+    yield(config)
+  end
+
+  def config
+    @config ||= Configuration.new
+  end
+end
+
+require_relative "active_record_proxy_adapters/railtie" if defined?(Rails::Railtie)

data/postgres_primary.dockerfile

@@ -0,0 +1,34 @@
+FROM docker.io/postgres:14-alpine
+
+ARG REPLICA_USER=replicator
+ARG REPLICA_PASSWORD=replicator
+ARG REPLICATION_SLOT_NAME=replication_slot
+ARG INIT_SQL=00_init.sql
+ARG POSTGRES_LOGGING_COLLECTOR=
+ARG POSTGRES_LOG_DESTINATION=
+ARG POSTGRES_LOG_STATEMENT=
+ENV CONF_SAMPLE="/usr/local/share/postgresql/postgresql.conf.sample"
+
+WORKDIR /docker-entrypoint-initdb.d
+
+USER root
+
+RUN touch $INIT_SQL
+RUN chown -R postgres:postgres $INIT_SQL
+RUN echo "CREATE USER ${REPLICA_USER} WITH REPLICATION ENCRYPTED PASSWORD '${REPLICA_PASSWORD}';" > $INIT_SQL
+RUN echo "SELECT pg_create_physical_replication_slot('${REPLICATION_SLOT_NAME}');" >> $INIT_SQL
+
+# Enable logging collector if given
+RUN if [[ ! -z "${POSTGRES_LOGGING_COLLECTOR}" ]]; then sed -i "s/#\(logging_collector = \)off\(.*\)/\1${POSTGRES_LOGGING_COLLECTOR}\2/" ${CONF_SAMPLE}; fi
+
+# Override  default log destination if given
+RUN if [[ ! -z "${POSTGRES_LOG_DESTINATION}" ]]; then sed -i "s/#\(log_destination = \)'stderr'\(.*\)/\1'${POSTGRES_LOG_DESTINATION}'\2/" ${CONF_SAMPLE}; fi
+
+# Override log statement if given
+RUN if [[ ! -z "${POSTGRES_LOG_STATEMENT}" ]]; then sed -i "s/#\(log_statement = \)'none'\(.*\)/\1'${POSTGRES_LOG_STATEMENT}'\2/" ${CONF_SAMPLE}; fi
+
+WORKDIR /
+
+USER postgres
+
+CMD  ["postgres", "-c", "wal_level=replica", "-c", "hot_standby=on", "-c", "max_wal_senders=10", "-c", "max_replication_slots=10", "-c", "hot_standby_feedback=on" ]

data/postgres_replica.dockerfile

@@ -0,0 +1,23 @@
+FROM docker.io/postgres:14-alpine
+
+ENV PRIMARY_DATABASE_HOST=localhost
+ENV PRIMARY_DATABASE_PORT=5432
+ENV PRIMARY_REPLICATION_SLOT=replication_slot
+
+USER root
+RUN printf '' > cmd.sh
+
+RUN echo 'until pg_basebackup --pgdata=/var/lib/postgresql/data -R --slot=$PRIMARY_REPLICATION_SLOT --host=$PRIMARY_DATABASE_HOST --port=$PRIMARY_DATABASE_PORT' >> cmd.sh
+RUN echo 'do' >> cmd.sh
+RUN echo "echo 'Waiting for primary to connect...'" >> cmd.sh
+RUN echo 'sleep 1s' >> cmd.sh
+RUN echo 'done' >> cmd.sh
+RUN echo "echo 'Backup done, starting replica...'" >> cmd.sh
+RUN echo 'chmod 0700 /var/lib/postgresql/data' >> cmd.sh
+RUN echo 'postgres' >> cmd.sh
+
+RUN chown -R postgres:postgres cmd.sh
+USER postgres
+RUN chmod u+rwx cmd.sh
+
+CMD [ "./cmd.sh" ]

data/sig/active_record_proxy_adapters.rbs

@@ -0,0 +1,4 @@
+module ActiveRecordProxyAdapters
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end