pg 1.1.0.pre20180730171000 → 1.1.0

data/sample/replication_monitor.rb

@@ -1,222 +0,0 @@
-# -*- ruby -*-
-# vim: set noet nosta sw=4 ts=4 :
-#
-# Get the current WAL segment and offset from a master postgresql
-# server, and compare slave servers to see how far behind they
-# are in MB.  This script should be easily modified for use with
-# Nagios/Mon/Monit/Zabbix/whatever, or wrapping it in a display loop,
-# and is suitable for both WAL shipping or streaming forms of replication.
-#
-# Mahlon E. Smith <mahlon@martini.nu>
-#
-# First argument is the master server, all other arguments are treated
-# as slave machines.
-#
-#	db_replication.monitor db-master.example.com ...
-#
-
-require 'ostruct'
-require 'optparse'
-require 'pathname'
-require 'etc'
-require 'pg'
-require 'pp'
-
-
-### A class to encapsulate the PG handles.
-###
-class PGMonitor
-
-	VERSION = %q$Id$
-
-	# When to consider a slave as 'behind', measured in WAL segments.
-	# The default WAL segment size is 16, so we'll alert after
-	# missing two WAL files worth of data.
-	#
-	LAG_ALERT = 32
-
-	### Create a new PGMonitor object.
-	###
-	def initialize( opts, hosts )
-		@opts        = opts
-		@master      = hosts.shift
-		@slaves      = hosts
-		@current_wal = {}
-		@failures    = []
-	end
-
-	attr_reader :opts, :current_wal, :master, :slaves, :failures
-
-
-	### Perform the connections and check the lag.
-	###
-	def check
-		# clear prior failures, get current xlog info
-		@failures = []
-		return unless self.get_current_wal
-
-		# check all slaves
-		self.slaves.each do |slave|
-			begin
-				slave_db = PG.connect(
-					:dbname   => self.opts.database,
-					:host     => slave,
-					:port     => self.opts.port,
-					:user     => self.opts.user,
-					:password => self.opts.pass,
-					:sslmode  => 'prefer'
-				)
-
-				xlog = slave_db.exec( 'SELECT pg_last_xlog_receive_location()' ).getvalue( 0, 0 )
-				slave_db.close
-
-				lag_in_megs = ( self.find_lag( xlog ).to_f / 1024 / 1024 ).abs
-				if lag_in_megs >= LAG_ALERT
-					failures << { :host => slave,
-						:error => "%0.2fMB behind the master." % [ lag_in_megs ] }
-				end
-			rescue => err
-				failures << { :host => slave, :error => err.message }
-			end
-		end
-	end
-
-
-	#########
-	protected
-	#########
-
-	### Ask the master for the current xlog information, to compare
-	### to slaves.  Returns true on succcess.  On failure, populates
-	### the failures array and returns false.
-	###
-	def get_current_wal
-		master_db = PG.connect(
-			:dbname   => self.opts.database,
-			:host     => self.master,
-			:port     => self.opts.port,
-			:user     => self.opts.user,
-			:password => self.opts.pass,
-			:sslmode  => 'prefer'
-		)
-
-		self.current_wal[ :segbytes ] = master_db.exec( 'SHOW wal_segment_size' ).
-			getvalue( 0, 0 ).sub( /\D+/, '' ).to_i << 20
-
-		current = master_db.exec( 'SELECT pg_current_xlog_location()' ).getvalue( 0, 0 )
-		self.current_wal[ :segment ], self.current_wal[ :offset ] = current.split( /\// )
-
-		master_db.close
-		return true
-
-	# If we can't get any of the info from the master, then there is no
-	# point in a comparison with slaves.
-	#
-	rescue => err
-		self.failures << { :host => self.master,
-			:error => 'Unable to retrieve required info from the master (%s)' % [ err.message ] }
-
-		return false
-	end
-
-
-	### Given an +xlog+ position from a slave server, return
-	### the number of bytes the slave needs to replay before it
-	### is caught up to the master.
-	###
-	def find_lag( xlog )
-		s_segment, s_offset = xlog.split( /\// )
-		m_segment  = self.current_wal[ :segment ]
-		m_offset   = self.current_wal[ :offset ]
-		m_segbytes = self.current_wal[ :segbytes ]
-
-		return (( m_segment.hex - s_segment.hex ) * m_segbytes) + ( m_offset.hex - s_offset.hex )
-	end
-
-end
-
-
-### Parse command line arguments.  Return a struct of global options.
-###
-def parse_args( args )
-	options          = OpenStruct.new
-	options.database = 'postgres'
-	options.port     = 5432
-	options.user     = Etc.getpwuid( Process.uid ).name
-	options.sslmode  = 'prefer'
-
-	opts = OptionParser.new do |opts|
-		opts.banner = "Usage: #{$0} [options] <master> <slave> [slave2, slave3...]"
-
-		opts.separator ''
-		opts.separator 'Connection options:'
-
-		opts.on( '-d', '--database DBNAME',
-				"specify the database to connect to (default: \"#{options.database}\")" ) do |db|
-			options.database = db
-		end
-
-		opts.on( '-h', '--host HOSTNAME', 'database server host' ) do |host|
-			options.host = host
-		end
-
-		opts.on( '-p', '--port PORT', Integer,
-				"database server port (default: \"#{options.port}\")" ) do |port|
-			options.port = port
-		end
-
-		opts.on( '-U', '--user NAME',
-				"database user name (default: \"#{options.user}\")" ) do |user|
-			options.user = user
-		end
-
-		opts.on( '-W', 'force password prompt' ) do |pw|
-			print 'Password: '
-			begin
-				system 'stty -echo'
-				options.pass = $stdin.gets.chomp
-			ensure
-				system 'stty echo'
-				puts
-			end
-		end
-
-		opts.separator ''
-		opts.separator 'Other options:'
-
-		opts.on_tail( '--help', 'show this help, then exit' ) do
-			$stderr.puts opts
-			exit
-		end
-
-		opts.on_tail( '--version', 'output version information, then exit' ) do
-			puts PGMonitor::VERSION
-			exit
-		end
-	end
-
-	opts.parse!( args )
-	return options
-end
-
-
-
-if __FILE__ == $0
-	opts = parse_args( ARGV )
-	raise ArgumentError, "At least two PostgreSQL servers are required." if ARGV.length < 2
-	mon = PGMonitor.new( opts, ARGV )
-
-	mon.check
-	if mon.failures.empty?
-		puts "All is well!"
-		exit 0
-	else
-		puts "Database replication delayed or broken."
-		mon.failures.each do |bad|
-			puts "%s: %s" % [ bad[ :host ], bad[ :error ] ]
-		end
-		exit 1
-	end
-end
-
-

data/sample/test_binary_values.rb

@@ -1,33 +0,0 @@
-# -*- ruby -*-1.9.1
-
-require 'pg'
-
-db = PG.connect( :dbname => 'test' )
-db.exec "DROP TABLE IF EXISTS test"
-db.exec "CREATE TABLE test (a INTEGER, b BYTEA)"
-
-a = 42
-b = [1, 2, 3]
-db.exec "INSERT INTO test(a, b) VALUES($1::int, $2::bytea)",
-	[a, {:value => b.pack('N*'), :format => 1}]
-
-db.exec( "SELECT a::int, b::bytea FROM test LIMIT 1", [], 1 ) do |res|
-
-	res.nfields.times do |i|
-		puts "Field %d is: %s, a %s (%s) column from table %p" % [
-			i,
-			res.fname( i ),
-			db.exec( "SELECT format_type($1,$2)", [res.ftype(i), res.fmod(1)] ).getvalue(0,0),
-			res.fformat( i ).zero? ? "string" : "binary",
-			res.ftable( i ),
-		]
-	end
-
-	res.each do |row|
-		puts "a = #{row['a'].inspect}"
-		puts "a (unpacked) = #{row['a'].unpack('N*').inspect}"
-		puts "b = #{row['b'].unpack('N*').inspect}"
-	end
-end
-
-

data/sample/wal_shipper.rb

@@ -1,434 +0,0 @@
-# -*- ruby -*-
-#
-# A script to wrap ssh and rsync for PostgreSQL WAL files shipping.
-# Mahlon E. Smith <mahlon@martini.nu>
-#
-# Based off of Joshua Drake's PITRTools concept, but with some important
-# differences:
-#
-#	- Only supports PostgreSQL >= 8.3
-#	- No support for rsync version < 3
-#	- Only shipping, no client side sync (too much opportunity for failure,
-#	  and it's easy to get a base backup manually)
-#	- WAL files are only stored once, regardless of how many
-#	  slaves are configured or not responding, and are removed from
-#	  the master when they are no longer needed.
-#	- Each slave can have completely distinct settings, instead
-#	  of a single set of options applied to all slaves
-#	- slave sync can be individually paused from the master
-#	- can run synchronously, or if you have a lot of slaves, threaded async mode
-#	- It's ruby, instead of python.  :)
-#
-# wal_shipper is configurable via an external YAML file, and will create
-# a template on its first run -- you'll need to modify it!  It expects
-# a directory structure like so:
-#
-#	postgres/
-#		data/...
-#		bin/wal_shipper.rb
-#		etc/wal_shipper.conf   <-- YAML settings!
-#		wal/
-#
-# It should be loaded from the PostgreSQL master's postgresql.conf
-# as such, after putting it into your postgres user homedir under 'bin':
-#
-#	archive_command = '/path/to/postgres_home/bin/wal_shipper.rb %p'
-#
-# Passwordless ssh keys need to be set up for the postgres user on all
-# participating masters and slaves.
-#
-# You can use any replay method of your choosing on the slaves.
-# Here's a nice example using pg_standby, to be put in data/recovery.conf:
-#
-#	restore_command = 'pg_standby -t /tmp/pgrecovery.done -s5 -w0 -c /path/to/postgres_home/wal_files/ %f %p %r'
-#
-# Or, here's another simple alternative data/recovery.conf, for using WAL shipping
-# alongside streaming replication:
-#
-#    standby_mode = 'on'
-#    primary_conninfo = 'host=master.example.com port=5432 user=repl password=XXXXXXX'
-#    restore_command = 'cp /usr/local/pgsql/wal/%f %p'
-#    trigger_file = '/usr/local/pgsql/pg.become_primary'
-#    archive_cleanup_command = '/usr/local/bin/pg_archivecleanup /usr/local/pgsql/wal %r'
-#
-#========================================================================================
-
-
-require 'pathname'
-require 'yaml'
-require 'fileutils'
-require 'ostruct'
-
-
-### Encapsulate WAL shipping functionality.
-###
-module WalShipper
-
-	### Send messages to the PostgreSQL log files.
-	###
-	def log( msg )
-		return unless @debug
-		puts "WAL Shipper: %s" % [ msg ]
-	end
-
-
-	### An object that represents a single destination from the
-	### configuration file.
-	###
-	class Destination < OpenStruct
-		include WalShipper
-
-		### Create a new WalShipper::Destination object.
-		def initialize( dest, debug=false )
-			@debug = debug
-			super( dest )
-			self.validate
-		end
-
-		#########
-		protected
-		#########
-
-
-		### Check for required keys and normalize various keys.
-		###
-		def validate
-			# Check for required destination keys
-			%w[ label kind ].each do |key|
-				if self.send( key.to_sym ).nil?
-					self.log "Destination %p missing required '%s' key." % [ self, key ]
-					self.invalid = true
-				end
-			end
-
-			# Ensure paths are Pathnames for the 'file' destination type.
-			self.path = Pathname.new( self.path ) if self.kind == 'file'
-
-			if self.kind == 'rsync-ssh'
-				self.port ||= 22
-				self.user = self.user ? "#{self.user}@" : ''
-			end
-		end
-	end # Class Destination
-
-
-
-	### Class for creating new Destination objects and determining how to
-	### ship WAL files to them.
-	###
-	class Dispatcher
-		include WalShipper
-
-		### Create a new Shipper object, given a +conf+ hash and a +wal+ file
-		### Pathname object.
-		###
-		def initialize( wal, conf )
-			# Make the config keys instance variables.
-			conf.each_pair {|key, val| self.instance_variable_set( "@#{key}", val ) }
-
-			# Spool directory check.
-			#
-			@spool = Pathname.new( @spool )
-			@spool.exist? or raise "The configured spool directory (%s) doesn't exist." % [ @spool ]
-
-			# Stop right away if we have disabled shipping.
-			#
-			unless @enabled
-				self.log "WAL shipping is disabled, queuing segment %s" % [ wal.basename ]
-				exit 1
-			end
-
-			# Instantiate Destination objects, creating new spool directories
-			# for each.
-			#
-			@destinations.
-				collect!{|dest| WalShipper::Destination.new( dest, @debug ) }.
-				reject  {|dest| dest.invalid }.
-				collect do |dest|
-					dest.spool = @spool + dest.label
-					dest.spool.mkdir( 0711 ) unless dest.spool.exist?
-					dest
-				end
-
-			# Put the WAL file into the spool for processing!
-			#
-			@waldir = @spool + 'wal_segments'
-			@waldir.mkdir( 0711 ) unless @waldir.exist?
-
-			self.log "Copying %s to %s" % [ wal.basename, @waldir ]
-			FileUtils::cp wal, @waldir
-
-			# 'wal' now references the copy.  The original is managed and auto-expired
-			# by PostgreSQL when a new checkpoint segment it reached.
-			@wal = @waldir + wal.basename
-		end
-
-
-		### Create hardlinks for the WAL file into each of the destination directories
-		### for separate queueing and recording of what was shipped successfully.
-		###
-		def link
-			@destinations.each do |dest|
-				self.log "Linking %s into %s" % [ @wal.basename, dest.spool.basename ]
-				FileUtils::ln @wal, dest.spool, :force => true
-			end
-		end
-
-
-		### Decide to be synchronous or threaded, and delegate each destination
-		### to the proper ship method.
-		###
-		def dispatch
-			# Synchronous mode.
-			#
-			unless @async
-				self.log "Performing a synchronous dispatch."
-				@destinations.each {|dest| self.dispatch_dest( dest ) }
-				return
-			end
-
-			tg = ThreadGroup.new
-
-			# Async, one thread per destination
-			#
-			if @async_max.nil? || @async_max.to_i.zero?
-				self.log "Performing an asynchronous dispatch: one thread per destination."
-				@destinations.each do |dest|
-					t = Thread.new do
-						Thread.current.abort_on_exception = true
-						self.dispatch_dest( dest )
-					end
-					tg.add( t )
-				end
-				tg.list.each {|t| t.join }
-				return
-			end
-
-			# Async, one thread per destination, in groups of asynx_max size.
-			#
-			self.log "Performing an asynchronous dispatch: one thread per destination, %d at a time." % [ @async_max ]
-			all_dests = @destinations.dup
-			dest_chunks = []
-			until all_dests.empty? do
-				dest_chunks << all_dests.slice!( 0, @async_max )
-			end
-
-			dest_chunks.each do |chunk|
-				chunk.each do |dest|
-					t = Thread.new do
-						Thread.current.abort_on_exception = true
-						self.dispatch_dest( dest )
-					end
-					tg.add( t )
-				end
-
-				tg.list.each {|t| t.join }
-			end
-
-			return
-		end
-
-
-		### Remove any WAL segments no longer needed by slaves.
-		###
-		def clean_spool
-			total = 0
-			@waldir.children.each do |wal|
-				if wal.stat.nlink == 1
-					total += wal.unlink
-				end
-			end
-
-			self.log "Removed %d WAL segment%s." % [ total, total == 1 ? '' : 's' ]
-		end
-
-
-
-		#########
-		protected
-		#########
-
-		### Send WAL segments to remote +dest+ via rsync+ssh.
-		### Passwordless keys between the user running this script (postmaster owner)
-		### and remote user need to be set up in advance.
-		###
-		def ship_rsync_ssh( dest )
-			if dest.host.nil?
-				self.log "Destination %p missing required 'host' key.  WAL is queued." % [ dest.host ]
-				return
-			end
-
-			rsync_flags = '-zc'
-			ssh_string = "%s -o ConnectTimeout=%d -o StrictHostKeyChecking=no -p %d" %
-				[ @ssh, @ssh_timeout || 10, dest.port ]
-			src_string = ''
-			dst_string = "%s%s:%s/" % [ dest.user, dest.host, dest.path ]
-
-			# If there are numerous files in the spool dir, it means there was
-			# an error transferring to this host in the past.  Try and ship all
-			# WAL segments, instead of just the new one.  PostgreSQL on the slave
-			# side will "do the right thing" as they come in, regardless of
-			# ordering.
-			#
-			if dest.spool.children.length > 1
-				src_string = dest.spool.to_s + '/'
-				rsync_flags << 'r'
-			else
-				src_string = dest.spool + @wal.basename
-			end
-
-
-			ship_wal_cmd = [
-				@rsync,
-				@debug ? (rsync_flags << 'vh') : (rsync_flags << 'q'),
-				'--remove-source-files',
-				'-e', ssh_string,
-				src_string, dst_string
-			]
-
-			self.log "Running command '%s'" % [ ship_wal_cmd.join(' ') ]
-			system *ship_wal_cmd
-
-			# Run external notification program on error, if one is configured.
-			#
-			unless $?.success?
-				self.log "Ack!  Error while shipping to %p, WAL is queued." % [ dest.label ]
-				system @error_cmd, dest.label if @error_cmd
-			end
-		end
-
-
-		### Copy WAL segments to remote path as set in +dest+.
-		### This is useful for longer term PITR, copying to NFS shares, etc.
-		###
-		def ship_file( dest )
-			if dest.path.nil?
-				self.log "Destination %p missing required 'path' key.  WAL is queued." % [ dest ]
-				return
-			end
-			dest.path.mkdir( 0711 ) unless dest.path.exist?
-
-			# If there are numerous files in the spool dir, it means there was
-			# an error transferring to this host in the past.  Try and ship all
-			# WAL segments, instead of just the new one.  PostgreSQL on the slave
-			# side will "do the right thing" as they come in, regardless of
-			# ordering.
-			#
-			if dest.spool.children.length > 1
-				dest.spool.children.each do |wal|
-					wal.unlink if self.copy_file( wal, dest.path, dest.label, dest.compress )
-				end
-			else
-				wal = dest.spool + @wal.basename
-				wal.unlink if self.copy_file( wal, dest.path, dest.label, dest.compress )
-			end
-		end
-
-
-		### Given a +wal+ Pathname, a +path+ destination, and the destination
-		### label, copy and optionally compress a WAL file.
-		###
-		def copy_file( wal, path, label, compress=false )
-			dest_file = path + wal.basename
-			FileUtils::cp wal, dest_file
-			if compress
-				system *[ 'gzip', '-f', dest_file ]
-				raise "Error while compressing: %s" % [ wal.basename ] unless $?.success?
-			end
-			self.log "Copied %s%s to %s." %
-				[ wal.basename, compress ? ' (and compressed)' : '', path ]
-			return true
-		rescue => err
-			self.log "Ack!  Error while copying '%s' (%s) to %p, WAL is queued." %
-				[ wal.basename, err.message, path ]
-			system @error_cmd, label if @error_cmd
-			return false
-		end
-
-
-		### Figure out how to send the WAL file to its intended destination +dest+.
-		###
-		def dispatch_dest( dest )
-			if ! dest.enabled.nil? && ! dest.enabled
-				self.log "Skipping explicity disabled destination %p, WAL is queued." % [ dest.label ]
-				return
-			end
-
-			# Send to the appropriate method.  ( rsync-ssh --> ship_rsync_ssh )
-			#
-			meth = ( 'ship_' + dest.kind.gsub(/-/, '_') ).to_sym
-			if WalShipper::Dispatcher.method_defined?( meth )
-				self.send( meth, dest )
-			else
-				self.log "Unknown destination kind %p for %p.  WAL is queued." % [ dest.kind, dest.label ]
-			end
-		end
-	end
-end
-
-# Ship the WAL file!
-#
-if __FILE__ == $0
-	CONFIG_DIR = Pathname.new( __FILE__ ).dirname.parent + 'etc'
-	CONFIG     = CONFIG_DIR + 'wal_shipper.conf'
-
-	unless CONFIG.exist?
-		CONFIG_DIR.mkdir( 0711 ) unless CONFIG_DIR.exist?
-		CONFIG.open('w') {|conf| conf.print(DATA.read) }
-		CONFIG.chmod( 0644 )
-		puts "No WAL shipping configuration found, default file created."
-	end
-
-	wal  = ARGV[0] or raise "No WAL file was specified on the command line."
-	wal  = Pathname.new( wal )
-	conf = YAML.load( CONFIG.read )
-
-	shipper = WalShipper::Dispatcher.new( wal, conf )
-	shipper.link
-	shipper.dispatch
-	shipper.clean_spool
-end
-
-
-__END__
----
-# Spool from pg_xlog to the working area?
-# This must be set to 'true' for wal shipping to function!
-enabled: false
-
-# Log everything to the PostgreSQL log files?
-debug: true
-
-# The working area for WAL segments.
-spool: /opt/local/var/db/postgresql84/wal
-
-# With multiple slaves, ship WAL in parallel, or be synchronous?
-async: false
-
-# Put a ceiling on the parallel threads?
-# '0' or removing this option uses a thread for each destination,
-# regardless of how many you have.  Keep in mind that's 16 * destination
-# count megs of simultaneous bandwidth.
-async_max: 5
-
-# Paths and settings for various binaries.
-rsync: /usr/bin/rsync
-ssh: /usr/bin/ssh
-ssh_timeout: 10
-
-destinations:
-
-- label: rsync-example
-  port: 2222
-  kind: rsync-ssh
-  host: localhost
-  user: postgres
-  path: wal    # relative to the user's homedir on the remote host
-  enabled: false
-
-- label: file-example
-  kind: file
-  compress: true
-  enabled: true
-  path: /tmp/someplace
-