prestogres 0.1.0

data/pgpool2/ps_status.c

@@ -0,0 +1,404 @@
+/* -*-pgsql-c-*- */
+/*
+ *
+ * $Header$
+ *
+ * This file was imported from PostgreSQL source code.
+ * See below for the copyright and description.
+ *
+ * pgpool: a language independent connection pool server for PostgreSQL
+ * written by Tatsuo Ishii
+ *
+ * Portions Copyright (c) 2003-2012	PgPool Global Development Group
+ *
+ */
+/*--------------------------------------------------------------------
+ * ps_status.c
+ *
+ * Routines to support changing the ps display of PostgreSQL backends
+ * to contain some useful information. Mechanism differs wildly across
+ * platforms.
+ *
+ * $PostgreSQL: pgsql/src/backend/utils/misc/ps_status.c,v 1.33 2006/10/04 00:30:04 momjian Exp $
+ *
+ * Copyright (c) 2000-2006, PostgreSQL Global Development Group
+ * various details abducted from various places
+ *--------------------------------------------------------------------
+ */
+
+#include <unistd.h>
+#ifdef HAVE_SYS_PSTAT_H
+#include <sys/pstat.h>			/* for HP-UX */
+#endif
+#ifdef HAVE_PS_STRINGS
+#include <machine/vmparam.h>	/* for old BSD */
+#include <sys/exec.h>
+#endif
+#if defined(__darwin__)
+#include <crt_externs.h>
+#endif
+
+#include "pool.h"
+#include <stdlib.h>
+#include <string.h>
+
+extern char **environ;
+bool		update_process_title = true;
+
+
+/*
+ * Alternative ways of updating ps display:
+ *
+ * PS_USE_SETPROCTITLE
+ *	   use the function setproctitle(const char *, ...)
+ *	   (newer BSD systems)
+ * PS_USE_PSTAT
+ *	   use the pstat(PSTAT_SETCMD, )
+ *	   (HPUX)
+ * PS_USE_PS_STRINGS
+ *	   assign PS_STRINGS->ps_argvstr = "string"
+ *	   (some BSD systems)
+ * PS_USE_CHANGE_ARGV
+ *	   assign argv[0] = "string"
+ *	   (some other BSD systems)
+ * PS_USE_CLOBBER_ARGV
+ *	   write over the argv and environment area
+ *	   (most SysV-like systems)
+ * PS_USE_WIN32
+ *	   push the string out as the name of a Windows event
+ * PS_USE_NONE
+ *	   don't update ps display
+ *	   (This is the default, as it is safest.)
+ */
+#if defined(HAVE_SETPROCTITLE)
+#define PS_USE_SETPROCTITLE
+#elif defined(HAVE_PSTAT) && defined(PSTAT_SETCMD)
+#define PS_USE_PSTAT
+#elif defined(HAVE_PS_STRINGS)
+#define PS_USE_PS_STRINGS
+#elif (defined(BSD) || defined(__bsdi__) || defined(__hurd__)) && !defined(__darwin__)
+#define PS_USE_CHANGE_ARGV
+#elif defined(__linux__) || defined(_AIX) || defined(__sgi) || (defined(sun) && !defined(BSD)) || defined(ultrix) || defined(__ksr__) || defined(__osf__) || defined(__svr4__) || defined(__svr5__) || defined(__darwin__)
+#define PS_USE_CLOBBER_ARGV
+#elif defined(WIN32)
+#define PS_USE_WIN32
+#else
+#define PS_USE_NONE
+#endif
+
+
+/* Different systems want the buffer padded differently */
+#if defined(_AIX) || defined(__linux__) || defined(__svr4__)
+#define PS_PADDING '\0'
+#else
+#define PS_PADDING ' '
+#endif
+
+
+#ifndef PS_USE_CLOBBER_ARGV
+/* all but one options need a buffer to write their ps line in */
+#define PS_BUFFER_SIZE 256
+static char ps_buffer[PS_BUFFER_SIZE];
+static const size_t ps_buffer_size = PS_BUFFER_SIZE;
+#else							/* PS_USE_CLOBBER_ARGV */
+static char *ps_buffer;			/* will point to argv area */
+static size_t ps_buffer_size;	/* space determined at run time */
+#endif   /* PS_USE_CLOBBER_ARGV */
+
+static size_t ps_buffer_fixed_size;		/* size of the constant prefix */
+
+/* save the original argv[] location here */
+static int	save_argc;
+static char **save_argv;
+
+
+/*
+ * Call this early in startup to save the original argc/argv values.
+ * If needed, we make a copy of the original argv[] array to preserve it
+ * from being clobbered by subsequent ps_display actions.
+ *
+ * (The original argv[] will not be overwritten by this routine, but may be
+ * overwritten during init_ps_display.	Also, the physical location of the
+ * environment strings may be moved, so this should be called before any code
+ * that might try to hang onto a getenv() result.)
+ */
+char	  **
+save_ps_display_args(int argc, char **argv)
+{
+	save_argc = argc;
+	save_argv = argv;
+
+#if defined(PS_USE_CLOBBER_ARGV)
+
+	/*
+	 * If we're going to overwrite the argv area, count the available space.
+	 * Also move the environment to make additional room.
+	 */
+	{
+		char	   *end_of_area = NULL;
+		char	  **new_environ;
+		int			i;
+
+		/*
+		 * check for contiguous argv strings
+		 */
+		for (i = 0; i < argc; i++)
+		{
+			if (i == 0 || end_of_area + 1 == argv[i])
+				end_of_area = argv[i] + strlen(argv[i]);
+		}
+
+		if (end_of_area == NULL)	/* probably can't happen? */
+		{
+			ps_buffer = NULL;
+			ps_buffer_size = 0;
+			return argv;
+		}
+
+		/*
+		 * check for contiguous environ strings following argv
+		 */
+		for (i = 0; environ[i] != NULL; i++)
+		{
+			if (end_of_area + 1 == environ[i])
+				end_of_area = environ[i] + strlen(environ[i]);
+		}
+
+		ps_buffer = argv[0];
+		ps_buffer_size = end_of_area - argv[0];
+
+		/*
+		 * move the environment out of the way
+		 */
+		new_environ = (char **) malloc((i + 1) * sizeof(char *));
+		for (i = 0; environ[i] != NULL; i++)
+			new_environ[i] = strdup(environ[i]);
+		new_environ[i] = NULL;
+		environ = new_environ;
+	}
+#endif   /* PS_USE_CLOBBER_ARGV */
+
+#if defined(PS_USE_CHANGE_ARGV) || defined(PS_USE_CLOBBER_ARGV)
+
+	/*
+	 * If we're going to change the original argv[] then make a copy for
+	 * argument parsing purposes.
+	 *
+	 * (NB: do NOT think to remove the copying of argv[], even though
+	 * postmaster.c finishes looking at argv[] long before we ever consider
+	 * changing the ps display.  On some platforms, getopt() keeps pointers
+	 * into the argv array, and will get horribly confused when it is
+	 * re-called to analyze a subprocess' argument string if the argv storage
+	 * has been clobbered meanwhile.  Other platforms have other dependencies
+	 * on argv[].
+	 */
+	{
+		char	  **new_argv;
+		int			i;
+
+		new_argv = (char **) malloc((argc + 1) * sizeof(char *));
+		for (i = 0; i < argc; i++)
+			new_argv[i] = strdup(argv[i]);
+		new_argv[argc] = NULL;
+
+#if defined(__darwin__)
+
+		/*
+		 * Darwin (and perhaps other NeXT-derived platforms?) has a static
+		 * copy of the argv pointer, which we may fix like so:
+		 */
+		*_NSGetArgv() = new_argv;
+#endif
+
+		argv = new_argv;
+	}
+#endif   /* PS_USE_CHANGE_ARGV or PS_USE_CLOBBER_ARGV */
+
+	return argv;
+}
+
+/*
+ * Call this once during subprocess startup to set the identification
+ * values.	At this point, the original argv[] array may be overwritten.
+ */
+void
+init_ps_display(const char *username, const char *dbname,
+				const char *host_info, const char *initial_str)
+{
+#ifndef PS_USE_NONE
+	/* no ps display if you didn't call save_ps_display_args() */
+	if (!save_argv)
+		return;
+#ifdef PS_USE_CLOBBER_ARGV
+	/* If ps_buffer is a pointer, it might still be null */
+	if (!ps_buffer)
+		return;
+#endif
+
+	/*
+	 * Overwrite argv[] to point at appropriate space, if needed
+	 */
+
+#ifdef PS_USE_CHANGE_ARGV
+	save_argv[0] = ps_buffer;
+	save_argv[1] = NULL;
+#endif   /* PS_USE_CHANGE_ARGV */
+
+#ifdef PS_USE_CLOBBER_ARGV
+	{
+		int			i;
+
+		/* make extra argv slots point at end_of_area (a NUL) */
+		for (i = 1; i < save_argc; i++)
+			save_argv[i] = ps_buffer + ps_buffer_size;
+	}
+#endif   /* PS_USE_CLOBBER_ARGV */
+
+	/*
+	 * Make fixed prefix of ps display.
+	 */
+
+#ifdef PS_USE_SETPROCTITLE
+
+	/*
+	 * apparently setproctitle() already adds a `progname: ' prefix to the
+	 * ps line
+	 */
+	snprintf(ps_buffer, ps_buffer_size, "");
+#else
+	snprintf(ps_buffer, ps_buffer_size,
+			 "pgpool: ");
+#endif
+
+	ps_buffer_fixed_size = strlen(ps_buffer);
+
+	set_ps_display(initial_str, true);
+#endif   /* not PS_USE_NONE */
+}
+
+
+
+/*
+ * Call this to update the ps status display to a fixed prefix plus an
+ * indication of what you're currently doing passed in the argument.
+ */
+void
+set_ps_display(const char *activity, bool force)
+{
+
+	if (!force && !update_process_title)
+		return;
+
+#ifndef PS_USE_NONE
+
+#ifdef PS_USE_CLOBBER_ARGV
+	/* If ps_buffer is a pointer, it might still be null */
+	if (!ps_buffer)
+		return;
+#endif
+
+	/* Update ps_buffer to contain both fixed part and activity */
+	strlcpy(ps_buffer + ps_buffer_fixed_size, activity,
+			ps_buffer_size - ps_buffer_fixed_size);
+
+	/* Transmit new setting to kernel, if necessary */
+
+#ifdef PS_USE_SETPROCTITLE
+	setproctitle("%s", ps_buffer);
+#endif
+
+#ifdef PS_USE_PSTAT
+	{
+		union pstun pst;
+
+		pst.pst_command = ps_buffer;
+		pstat(PSTAT_SETCMD, pst, strlen(ps_buffer), 0, 0);
+	}
+#endif   /* PS_USE_PSTAT */
+
+#ifdef PS_USE_PS_STRINGS
+	PS_STRINGS->ps_nargvstr = 1;
+	PS_STRINGS->ps_argvstr = ps_buffer;
+#endif   /* PS_USE_PS_STRINGS */
+
+#ifdef PS_USE_CLOBBER_ARGV
+	{
+		int			buflen;
+
+		/* pad unused memory */
+		buflen = strlen(ps_buffer);
+		memset(ps_buffer + buflen, PS_PADDING, ps_buffer_size - buflen);
+	}
+#endif   /* PS_USE_CLOBBER_ARGV */
+
+#ifdef PS_USE_WIN32
+	{
+		/*
+		 * Win32 does not support showing any changed arguments. To make it at
+		 * all possible to track which backend is doing what, we create a
+		 * named object that can be viewed with for example Process Explorer.
+		 */
+		static HANDLE ident_handle = INVALID_HANDLE_VALUE;
+		char		name[PS_BUFFER_SIZE + 32];
+
+		if (ident_handle != INVALID_HANDLE_VALUE)
+			CloseHandle(ident_handle);
+
+		sprintf(name, "pgident: %s", ps_buffer);
+
+		ident_handle = CreateEvent(NULL, TRUE, FALSE, name);
+	}
+#endif   /* PS_USE_WIN32 */
+#endif   /* not PS_USE_NONE */
+}
+
+
+/*
+ * Returns what's currently in the ps display, in case someone needs
+ * it.	Note that only the activity part is returned.  On some platforms
+ * the string will not be null-terminated, so return the effective
+ * length into *displen.
+ */
+const char *
+get_ps_display(int *displen)
+{
+#ifdef PS_USE_CLOBBER_ARGV
+	size_t		offset;
+
+	/* If ps_buffer is a pointer, it might still be null */
+	if (!ps_buffer)
+	{
+		*displen = 0;
+		return "";
+	}
+
+	/* Remove any trailing spaces to offset the effect of PS_PADDING */
+	offset = ps_buffer_size;
+	while (offset > ps_buffer_fixed_size && ps_buffer[offset - 1] == PS_PADDING)
+		offset--;
+
+	*displen = offset - ps_buffer_fixed_size;
+#else
+	*displen = strlen(ps_buffer + ps_buffer_fixed_size);
+#endif
+
+	return ps_buffer + ps_buffer_fixed_size;
+}
+
+/*
+ * Show ps idle status
+ */
+void pool_ps_idle_display(POOL_CONNECTION_POOL *backend)
+{
+	StartupPacket *sp;
+	char psbuf[1024];
+
+	sp = MASTER_CONNECTION(backend)->sp;
+	if (MASTER(backend)->tstate == 'T')
+		snprintf(psbuf, sizeof(psbuf), "%s %s %s idle in transaction",
+				 sp->user, sp->database, remote_ps_data);
+	else
+		snprintf(psbuf, sizeof(psbuf), "%s %s %s idle",
+				 sp->user, sp->database, remote_ps_data);
+	set_ps_display(psbuf, false);
+}

data/pgpool2/recovery.c

@@ -0,0 +1,435 @@
+/* -*-pgsql-c-*- */
+/*
+ * $Header$
+ *
+ * pgpool: a language independent connection pool server for PostgreSQL
+ * written by Tatsuo Ishii
+ *
+ * Copyright (c) 2003-2013	PgPool Global Development Group
+ *
+ * Permission to use, copy, modify, and distribute this software and
+ * its documentation for any purpose and without fee is hereby
+ * granted, provided that the above copyright notice appear in all
+ * copies and that both that copyright notice and this permission
+ * notice appear in supporting documentation, and that the name of the
+ * author not be used in advertising or publicity pertaining to
+ * distribution of the software without specific, written prior
+ * permission. The author makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * recovery.c: online recovery process
+ *
+ */
+
+#include "config.h"
+
+#include <unistd.h>
+#include <string.h>
+
+#include "pool.h"
+#include "pool_config.h"
+
+#include "libpq-fe.h"
+
+#include "watchdog/watchdog.h"
+#include "watchdog/wd_ext.h"
+
+#define WAIT_RETRY_COUNT (pool_config->recovery_timeout / 3)
+
+#define FIRST_STAGE 0
+#define SECOND_STAGE 1
+
+static int exec_checkpoint(PGconn *conn);
+static int exec_recovery(PGconn *conn, BackendInfo *backend, char stage);
+static int exec_remote_start(PGconn *conn, BackendInfo *backend);
+static PGconn *connect_backend_libpq(BackendInfo *backend);
+static int check_postmaster_started(BackendInfo *backend);
+
+static char recovery_command[1024];
+
+extern volatile sig_atomic_t pcp_wakeup_request;
+
+/*
+ * Start online recovery.
+ * "recovery_node" is the node to be recovered.
+ * Master or primary node is chosen in this function.
+ */
+int start_recovery(int recovery_node)
+{
+	int node_id;
+	BackendInfo *backend;
+	BackendInfo *recovery_backend;
+	PGconn *conn;
+	int failback_wait_count;
+#define FAILBACK_WAIT_MAX_RETRY 5		/* 5 seconds should be enough for failback operation */
+
+	pool_log("starting recovering node %d", recovery_node);
+
+	if ( (recovery_node < 0) || (recovery_node >= pool_config->backend_desc->num_backends) )
+	{
+		pool_error("start_recovery: node id %d is not valid", recovery_node);
+		return 1;
+	}
+
+	if (VALID_BACKEND(recovery_node))
+	{
+
+		pool_error("start_recovery: backend node %d is alive", recovery_node);
+		return 1;
+	}
+
+	Req_info->kind = NODE_RECOVERY_REQUEST;
+
+	/* select master/primary node */
+	node_id = MASTER_SLAVE ? PRIMARY_NODE_ID : REAL_MASTER_NODE_ID;
+	backend = &pool_config->backend_desc->backend_info[node_id];
+
+	/* get node info to be recovered */
+	recovery_backend = &pool_config->backend_desc->backend_info[recovery_node];
+
+	conn = connect_backend_libpq(backend);
+	if (conn == NULL)
+	{
+		pool_error("start_recovery: could not connect master node (%d)", node_id);
+		return 1;
+	}
+
+	/* 1st stage */
+	if (REPLICATION)
+	{
+		if (exec_checkpoint(conn) != 0)
+		{
+			PQfinish(conn);
+			pool_error("start_recovery: CHECKPOINT failed");
+			return 1;
+		}
+		pool_log("CHECKPOINT in the 1st stage done");
+	}
+
+	if (exec_recovery(conn, recovery_backend, FIRST_STAGE) != 0)
+	{
+		PQfinish(conn);
+		return 1;
+	}
+
+	pool_log("1st stage is done");
+
+	if (REPLICATION)
+	{
+		pool_log("starting 2nd stage");
+
+		/* 2nd stage */
+		*InRecovery = RECOVERY_ONLINE;
+		if (pool_config->use_watchdog)
+		{
+			/* announce start recovery */
+			if (WD_OK != wd_start_recovery())
+			{
+				PQfinish(conn);
+				pool_error("start_recovery: timeover for waiting connection closed in the other pgpools");
+				return 1;
+			}
+		}
+
+		if (wait_connection_closed() != 0)
+		{
+			PQfinish(conn);
+			pool_error("start_recovery: timeover for waiting connection closed");
+			return 1;
+		}
+
+		pool_log("all connections from clients have been closed");
+
+		if (exec_checkpoint(conn) != 0)
+		{
+			PQfinish(conn);
+			pool_error("start_recovery: CHECKPOINT failed");
+			return 1;
+		}
+
+		pool_log("CHECKPOINT in the 2nd stage done");
+
+		if (exec_recovery(conn, recovery_backend, SECOND_STAGE) != 0)
+		{
+			PQfinish(conn);
+			return 1;
+		}
+	}
+
+	if (exec_remote_start(conn, recovery_backend) != 0)
+	{
+		PQfinish(conn);
+		pool_error("start_recovery: remote start failed");
+		return 1;
+	}
+
+	if (check_postmaster_started(recovery_backend))
+	{
+		PQfinish(conn);
+		pool_error("start_recovery: check start failed");
+		return 1;
+	}
+
+	pool_log("%d node restarted", recovery_node);
+
+	/*
+	 * reset failover completion flag.  this is necessary since
+	 * previous failover/failback will set the flag to 1.
+	 */
+	pcp_wakeup_request = 0;
+
+	/* send failback request to pgpool parent */
+	send_failback_request(recovery_node);
+
+	/* wait for failback */
+	failback_wait_count = 0;
+	while (!pcp_wakeup_request)
+	{
+		struct timeval t = {1, 0};
+		/* polling SIGUSR2 signal every 1 sec */
+		select(0, NULL, NULL, NULL, &t);
+		failback_wait_count++;
+		if (failback_wait_count >= FAILBACK_WAIT_MAX_RETRY)
+		{
+			pool_log("start_recovery: waiting for wake up request is timeout(%d seconds)",
+					 FAILBACK_WAIT_MAX_RETRY);
+			break;
+		}
+	}
+	pcp_wakeup_request = 0;
+
+	PQfinish(conn);
+
+	pool_log("recovery done");
+
+	return 0;
+}
+
+/*
+ * Notice all children finishing recovery.
+ */
+void finish_recovery(void)
+{
+	/* announce end recovery */
+	if (pool_config->use_watchdog && *InRecovery != RECOVERY_INIT)
+	{
+		wd_end_recovery();
+	}
+
+	*InRecovery = RECOVERY_INIT;
+	kill(getppid(), SIGUSR2);
+}
+
+/*
+ * Execute CHECKPOINT
+ */
+static int exec_checkpoint(PGconn *conn)
+{
+	PGresult *result;
+	int r;
+
+	pool_debug("exec_checkpoint: start checkpoint");
+	result = PQexec(conn, "CHECKPOINT");
+	r = (PQresultStatus(result) !=  PGRES_COMMAND_OK);
+	PQclear(result);
+	pool_debug("exec_checkpoint: finish checkpoint");
+	return r;
+}
+
+/*
+ * Call pgpool_recovery() function.
+ */
+static int exec_recovery(PGconn *conn, BackendInfo *backend, char stage)
+{
+	PGresult *result;
+	char *hostname;
+	char *script;
+	int r;
+
+	if (strlen(backend->backend_hostname) == 0 || *(backend->backend_hostname) == '/')
+		hostname = "localhost";
+	else
+		hostname = backend->backend_hostname;
+
+	script = (stage == FIRST_STAGE) ?
+		pool_config->recovery_1st_stage_command : pool_config->recovery_2nd_stage_command;
+
+	if (script == NULL || strlen(script) == 0)
+	{
+		/* do not execute script */
+		return 0;
+	}
+
+	snprintf(recovery_command,
+			 sizeof(recovery_command),
+			 "SELECT pgpool_recovery('%s', '%s', '%s')",
+			 script,
+			 hostname,
+			 backend->backend_data_directory);
+
+	pool_log("starting recovery command: \"%s\"", recovery_command);
+
+	pool_debug("exec_recovery: start recovery");
+	result = PQexec(conn, recovery_command);
+	r = (PQresultStatus(result) !=  PGRES_TUPLES_OK);
+	if (r != 0)
+	{
+		pool_error("exec_recovery: %s command failed at %s",
+				   script,
+				   (stage == FIRST_STAGE) ? "1st stage" : "2nd stage");
+	}
+	PQclear(result);
+	pool_debug("exec_recovery: finish recovery");
+	return r;
+}
+
+/*
+ * Call pgpool_remote_start() function.
+ */
+static int exec_remote_start(PGconn *conn, BackendInfo *backend)
+{
+	PGresult *result;
+	char *hostname;
+	int r;
+
+	if (strlen(backend->backend_hostname) == 0 || *(backend->backend_hostname) == '/')
+		hostname = "localhost";
+	else
+		hostname = backend->backend_hostname;
+
+	snprintf(recovery_command, sizeof(recovery_command),
+			 "SELECT pgpool_remote_start('%s', '%s')",
+			 hostname,
+			 backend->backend_data_directory);
+
+	pool_debug("exec_remote_start: start pgpool_remote_start");
+	result = PQexec(conn, recovery_command);
+	r = (PQresultStatus(result) !=  PGRES_TUPLES_OK);
+	if (r != 0)
+		pool_error("exec_remote_start: pgpool_remote_start failed: %s", PQresultErrorMessage(result));
+	PQclear(result);
+	pool_debug("exec_remote_start: finish pgpool_remote_start");
+	return r;
+}
+
+/*
+ * Check postmaster is started.
+ */
+static int check_postmaster_started(BackendInfo *backend)
+{
+	int i = 0;
+	char port_str[16];
+	PGconn *conn;
+	char *dbname;
+
+	snprintf(port_str, sizeof(port_str),"%d", backend->backend_port);
+
+	/*
+	 * First we try with "postgres" database.
+	 */
+	dbname = "postgres";
+
+	do {
+		ConnStatusType r;
+
+		pool_log("check_postmaster_started: try to connect to postmaster on hostname:%s database:%s user:%s (retry %d times)",
+				 backend->backend_hostname, dbname, pool_config->recovery_user, i);
+
+		conn = PQsetdbLogin(backend->backend_hostname,
+							port_str,
+							NULL,
+							NULL,
+							dbname,
+							pool_config->recovery_user,
+							pool_config->recovery_password);
+
+		r = PQstatus(conn);
+		PQfinish(conn);
+		if (r == CONNECTION_OK)
+			return 0;
+
+		pool_log("check_postmaster_started: failed to connect to postmaster on hostname:%s database:%s user:%s",
+			 backend->backend_hostname, dbname, pool_config->recovery_user);
+		
+		sleep(3);
+	} while (i++ < 3);	/* XXX Hard coded retry (9 seconds) */
+
+	/*
+	 * Retry with "template1" database.
+	 */
+	dbname = "template1";
+	i = 0;
+
+	do {
+		ConnStatusType r;
+
+		pool_log("check_postmaster_started: try to connect to postmaster on hostname:%s database:%s user:%s (retry %d times)",
+				 backend->backend_hostname, dbname, pool_config->recovery_user, i);
+
+		conn = PQsetdbLogin(backend->backend_hostname,
+							port_str,
+							NULL,
+							NULL,
+							dbname,
+							pool_config->recovery_user,
+							pool_config->recovery_password);
+
+		r = PQstatus(conn);
+		PQfinish(conn);
+		if (r == CONNECTION_OK)
+			return 0;
+
+		pool_log("check_postmaster_started: failed to connect to postmaster on hostname:%s database:%s user:%s",
+			 backend->backend_hostname, dbname, pool_config->recovery_user);
+
+		if (WAIT_RETRY_COUNT != 0)
+			sleep(3);
+	} while (i++ < WAIT_RETRY_COUNT);
+
+	pool_error("check_postmaster_started: remote host start up did not finish in %d sec.", pool_config->recovery_timeout);
+	return 1;
+}
+
+static PGconn *connect_backend_libpq(BackendInfo *backend)
+{
+	char port_str[16];
+	PGconn *conn;
+
+	snprintf(port_str, sizeof(port_str),
+			 "%d", backend->backend_port);
+	conn = PQsetdbLogin(backend->backend_hostname,
+						port_str,
+						NULL,
+						NULL,
+						"template1",
+						pool_config->recovery_user,
+						pool_config->recovery_password);
+
+	if (PQstatus(conn) != CONNECTION_OK)
+	{
+		PQfinish(conn);
+		return NULL;
+	}
+	return conn;
+}
+
+/*
+ * Wait all connections are closed.
+ */
+int wait_connection_closed(void)
+{
+	int i = 0;
+
+	do {
+
+		if (Req_info->conn_counter == 0)
+			return 0;
+
+		if (WAIT_RETRY_COUNT != 0)
+			sleep(3);
+	} while (i++ < WAIT_RETRY_COUNT);
+
+	pool_error("wait_connection_closed: existing connections did not close in %d sec.", pool_config->recovery_timeout);
+	return 1;
+}