pg 1.1.0.pre20180730144600 → 1.1.0.pre20180730171000
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/.gems +6 -0
- data/.hgignore +21 -0
- data/.hgsigs +29 -0
- data/.hgtags +36 -0
- data/.hoerc +2 -0
- data/.irbrc +23 -0
- data/.pryrc +23 -0
- data/.tm_properties +21 -0
- data/.travis.yml +41 -0
- data/Gemfile +2 -0
- data/Manifest.txt +2 -65
- data/Rakefile +2 -1
- data/Rakefile.cross +3 -3
- data/appveyor.yml +50 -0
- data/certs/ged.pem +26 -0
- data/lib/pg.rb +1 -1
- data/lib/pg/binary_decoder.rb +22 -0
- data/lib/pg/tuple.rb +30 -0
- data/misc/openssl-pg-segfault.rb +31 -0
- data/misc/postgres/History.txt +9 -0
- data/misc/postgres/Manifest.txt +5 -0
- data/misc/postgres/README.txt +21 -0
- data/misc/postgres/Rakefile +21 -0
- data/misc/postgres/lib/postgres.rb +16 -0
- data/misc/ruby-pg/History.txt +9 -0
- data/misc/ruby-pg/Manifest.txt +5 -0
- data/misc/ruby-pg/README.txt +21 -0
- data/misc/ruby-pg/Rakefile +21 -0
- data/misc/ruby-pg/lib/ruby/pg.rb +16 -0
- data/pg.gemspec +61 -0
- data/sample/array_insert.rb +20 -0
- data/sample/async_api.rb +106 -0
- data/sample/async_copyto.rb +39 -0
- data/sample/async_mixed.rb +56 -0
- data/sample/check_conn.rb +21 -0
- data/sample/copydata.rb +71 -0
- data/sample/copyfrom.rb +81 -0
- data/sample/copyto.rb +19 -0
- data/sample/cursor.rb +21 -0
- data/sample/disk_usage_report.rb +177 -0
- data/sample/issue-119.rb +94 -0
- data/sample/losample.rb +69 -0
- data/sample/minimal-testcase.rb +17 -0
- data/sample/notify_wait.rb +72 -0
- data/sample/pg_statistics.rb +285 -0
- data/sample/replication_monitor.rb +222 -0
- data/sample/test_binary_values.rb +33 -0
- data/sample/wal_shipper.rb +434 -0
- data/sample/warehouse_partitions.rb +311 -0
- data/spec/pg/connection_sync_spec.rb +41 -0
- data/spec/pg/tuple_spec.rb +266 -0
- metadata +69 -25
- metadata.gz.sig +0 -0
- data/ChangeLog +0 -0
@@ -0,0 +1,222 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
# vim: set noet nosta sw=4 ts=4 :
|
3
|
+
#
|
4
|
+
# Get the current WAL segment and offset from a master postgresql
|
5
|
+
# server, and compare slave servers to see how far behind they
|
6
|
+
# are in MB. This script should be easily modified for use with
|
7
|
+
# Nagios/Mon/Monit/Zabbix/whatever, or wrapping it in a display loop,
|
8
|
+
# and is suitable for both WAL shipping or streaming forms of replication.
|
9
|
+
#
|
10
|
+
# Mahlon E. Smith <mahlon@martini.nu>
|
11
|
+
#
|
12
|
+
# First argument is the master server, all other arguments are treated
|
13
|
+
# as slave machines.
|
14
|
+
#
|
15
|
+
# db_replication.monitor db-master.example.com ...
|
16
|
+
#
|
17
|
+
|
18
|
+
require 'ostruct'
|
19
|
+
require 'optparse'
|
20
|
+
require 'pathname'
|
21
|
+
require 'etc'
|
22
|
+
require 'pg'
|
23
|
+
require 'pp'
|
24
|
+
|
25
|
+
|
26
|
+
### A class to encapsulate the PG handles.
|
27
|
+
###
|
28
|
+
class PGMonitor
|
29
|
+
|
30
|
+
VERSION = %q$Id$
|
31
|
+
|
32
|
+
# When to consider a slave as 'behind', measured in WAL segments.
|
33
|
+
# The default WAL segment size is 16, so we'll alert after
|
34
|
+
# missing two WAL files worth of data.
|
35
|
+
#
|
36
|
+
LAG_ALERT = 32
|
37
|
+
|
38
|
+
### Create a new PGMonitor object.
|
39
|
+
###
|
40
|
+
def initialize( opts, hosts )
|
41
|
+
@opts = opts
|
42
|
+
@master = hosts.shift
|
43
|
+
@slaves = hosts
|
44
|
+
@current_wal = {}
|
45
|
+
@failures = []
|
46
|
+
end
|
47
|
+
|
48
|
+
attr_reader :opts, :current_wal, :master, :slaves, :failures
|
49
|
+
|
50
|
+
|
51
|
+
### Perform the connections and check the lag.
|
52
|
+
###
|
53
|
+
def check
|
54
|
+
# clear prior failures, get current xlog info
|
55
|
+
@failures = []
|
56
|
+
return unless self.get_current_wal
|
57
|
+
|
58
|
+
# check all slaves
|
59
|
+
self.slaves.each do |slave|
|
60
|
+
begin
|
61
|
+
slave_db = PG.connect(
|
62
|
+
:dbname => self.opts.database,
|
63
|
+
:host => slave,
|
64
|
+
:port => self.opts.port,
|
65
|
+
:user => self.opts.user,
|
66
|
+
:password => self.opts.pass,
|
67
|
+
:sslmode => 'prefer'
|
68
|
+
)
|
69
|
+
|
70
|
+
xlog = slave_db.exec( 'SELECT pg_last_xlog_receive_location()' ).getvalue( 0, 0 )
|
71
|
+
slave_db.close
|
72
|
+
|
73
|
+
lag_in_megs = ( self.find_lag( xlog ).to_f / 1024 / 1024 ).abs
|
74
|
+
if lag_in_megs >= LAG_ALERT
|
75
|
+
failures << { :host => slave,
|
76
|
+
:error => "%0.2fMB behind the master." % [ lag_in_megs ] }
|
77
|
+
end
|
78
|
+
rescue => err
|
79
|
+
failures << { :host => slave, :error => err.message }
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
|
85
|
+
#########
|
86
|
+
protected
|
87
|
+
#########
|
88
|
+
|
89
|
+
### Ask the master for the current xlog information, to compare
|
90
|
+
### to slaves. Returns true on succcess. On failure, populates
|
91
|
+
### the failures array and returns false.
|
92
|
+
###
|
93
|
+
def get_current_wal
|
94
|
+
master_db = PG.connect(
|
95
|
+
:dbname => self.opts.database,
|
96
|
+
:host => self.master,
|
97
|
+
:port => self.opts.port,
|
98
|
+
:user => self.opts.user,
|
99
|
+
:password => self.opts.pass,
|
100
|
+
:sslmode => 'prefer'
|
101
|
+
)
|
102
|
+
|
103
|
+
self.current_wal[ :segbytes ] = master_db.exec( 'SHOW wal_segment_size' ).
|
104
|
+
getvalue( 0, 0 ).sub( /\D+/, '' ).to_i << 20
|
105
|
+
|
106
|
+
current = master_db.exec( 'SELECT pg_current_xlog_location()' ).getvalue( 0, 0 )
|
107
|
+
self.current_wal[ :segment ], self.current_wal[ :offset ] = current.split( /\// )
|
108
|
+
|
109
|
+
master_db.close
|
110
|
+
return true
|
111
|
+
|
112
|
+
# If we can't get any of the info from the master, then there is no
|
113
|
+
# point in a comparison with slaves.
|
114
|
+
#
|
115
|
+
rescue => err
|
116
|
+
self.failures << { :host => self.master,
|
117
|
+
:error => 'Unable to retrieve required info from the master (%s)' % [ err.message ] }
|
118
|
+
|
119
|
+
return false
|
120
|
+
end
|
121
|
+
|
122
|
+
|
123
|
+
### Given an +xlog+ position from a slave server, return
|
124
|
+
### the number of bytes the slave needs to replay before it
|
125
|
+
### is caught up to the master.
|
126
|
+
###
|
127
|
+
def find_lag( xlog )
|
128
|
+
s_segment, s_offset = xlog.split( /\// )
|
129
|
+
m_segment = self.current_wal[ :segment ]
|
130
|
+
m_offset = self.current_wal[ :offset ]
|
131
|
+
m_segbytes = self.current_wal[ :segbytes ]
|
132
|
+
|
133
|
+
return (( m_segment.hex - s_segment.hex ) * m_segbytes) + ( m_offset.hex - s_offset.hex )
|
134
|
+
end
|
135
|
+
|
136
|
+
end
|
137
|
+
|
138
|
+
|
139
|
+
### Parse command line arguments. Return a struct of global options.
|
140
|
+
###
|
141
|
+
def parse_args( args )
|
142
|
+
options = OpenStruct.new
|
143
|
+
options.database = 'postgres'
|
144
|
+
options.port = 5432
|
145
|
+
options.user = Etc.getpwuid( Process.uid ).name
|
146
|
+
options.sslmode = 'prefer'
|
147
|
+
|
148
|
+
opts = OptionParser.new do |opts|
|
149
|
+
opts.banner = "Usage: #{$0} [options] <master> <slave> [slave2, slave3...]"
|
150
|
+
|
151
|
+
opts.separator ''
|
152
|
+
opts.separator 'Connection options:'
|
153
|
+
|
154
|
+
opts.on( '-d', '--database DBNAME',
|
155
|
+
"specify the database to connect to (default: \"#{options.database}\")" ) do |db|
|
156
|
+
options.database = db
|
157
|
+
end
|
158
|
+
|
159
|
+
opts.on( '-h', '--host HOSTNAME', 'database server host' ) do |host|
|
160
|
+
options.host = host
|
161
|
+
end
|
162
|
+
|
163
|
+
opts.on( '-p', '--port PORT', Integer,
|
164
|
+
"database server port (default: \"#{options.port}\")" ) do |port|
|
165
|
+
options.port = port
|
166
|
+
end
|
167
|
+
|
168
|
+
opts.on( '-U', '--user NAME',
|
169
|
+
"database user name (default: \"#{options.user}\")" ) do |user|
|
170
|
+
options.user = user
|
171
|
+
end
|
172
|
+
|
173
|
+
opts.on( '-W', 'force password prompt' ) do |pw|
|
174
|
+
print 'Password: '
|
175
|
+
begin
|
176
|
+
system 'stty -echo'
|
177
|
+
options.pass = $stdin.gets.chomp
|
178
|
+
ensure
|
179
|
+
system 'stty echo'
|
180
|
+
puts
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
opts.separator ''
|
185
|
+
opts.separator 'Other options:'
|
186
|
+
|
187
|
+
opts.on_tail( '--help', 'show this help, then exit' ) do
|
188
|
+
$stderr.puts opts
|
189
|
+
exit
|
190
|
+
end
|
191
|
+
|
192
|
+
opts.on_tail( '--version', 'output version information, then exit' ) do
|
193
|
+
puts PGMonitor::VERSION
|
194
|
+
exit
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
opts.parse!( args )
|
199
|
+
return options
|
200
|
+
end
|
201
|
+
|
202
|
+
|
203
|
+
|
204
|
+
if __FILE__ == $0
|
205
|
+
opts = parse_args( ARGV )
|
206
|
+
raise ArgumentError, "At least two PostgreSQL servers are required." if ARGV.length < 2
|
207
|
+
mon = PGMonitor.new( opts, ARGV )
|
208
|
+
|
209
|
+
mon.check
|
210
|
+
if mon.failures.empty?
|
211
|
+
puts "All is well!"
|
212
|
+
exit 0
|
213
|
+
else
|
214
|
+
puts "Database replication delayed or broken."
|
215
|
+
mon.failures.each do |bad|
|
216
|
+
puts "%s: %s" % [ bad[ :host ], bad[ :error ] ]
|
217
|
+
end
|
218
|
+
exit 1
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# -*- ruby -*-1.9.1
|
2
|
+
|
3
|
+
require 'pg'
|
4
|
+
|
5
|
+
db = PG.connect( :dbname => 'test' )
|
6
|
+
db.exec "DROP TABLE IF EXISTS test"
|
7
|
+
db.exec "CREATE TABLE test (a INTEGER, b BYTEA)"
|
8
|
+
|
9
|
+
a = 42
|
10
|
+
b = [1, 2, 3]
|
11
|
+
db.exec "INSERT INTO test(a, b) VALUES($1::int, $2::bytea)",
|
12
|
+
[a, {:value => b.pack('N*'), :format => 1}]
|
13
|
+
|
14
|
+
db.exec( "SELECT a::int, b::bytea FROM test LIMIT 1", [], 1 ) do |res|
|
15
|
+
|
16
|
+
res.nfields.times do |i|
|
17
|
+
puts "Field %d is: %s, a %s (%s) column from table %p" % [
|
18
|
+
i,
|
19
|
+
res.fname( i ),
|
20
|
+
db.exec( "SELECT format_type($1,$2)", [res.ftype(i), res.fmod(1)] ).getvalue(0,0),
|
21
|
+
res.fformat( i ).zero? ? "string" : "binary",
|
22
|
+
res.ftable( i ),
|
23
|
+
]
|
24
|
+
end
|
25
|
+
|
26
|
+
res.each do |row|
|
27
|
+
puts "a = #{row['a'].inspect}"
|
28
|
+
puts "a (unpacked) = #{row['a'].unpack('N*').inspect}"
|
29
|
+
puts "b = #{row['b'].unpack('N*').inspect}"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
|
@@ -0,0 +1,434 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
#
|
3
|
+
# A script to wrap ssh and rsync for PostgreSQL WAL files shipping.
|
4
|
+
# Mahlon E. Smith <mahlon@martini.nu>
|
5
|
+
#
|
6
|
+
# Based off of Joshua Drake's PITRTools concept, but with some important
|
7
|
+
# differences:
|
8
|
+
#
|
9
|
+
# - Only supports PostgreSQL >= 8.3
|
10
|
+
# - No support for rsync version < 3
|
11
|
+
# - Only shipping, no client side sync (too much opportunity for failure,
|
12
|
+
# and it's easy to get a base backup manually)
|
13
|
+
# - WAL files are only stored once, regardless of how many
|
14
|
+
# slaves are configured or not responding, and are removed from
|
15
|
+
# the master when they are no longer needed.
|
16
|
+
# - Each slave can have completely distinct settings, instead
|
17
|
+
# of a single set of options applied to all slaves
|
18
|
+
# - slave sync can be individually paused from the master
|
19
|
+
# - can run synchronously, or if you have a lot of slaves, threaded async mode
|
20
|
+
# - It's ruby, instead of python. :)
|
21
|
+
#
|
22
|
+
# wal_shipper is configurable via an external YAML file, and will create
|
23
|
+
# a template on its first run -- you'll need to modify it! It expects
|
24
|
+
# a directory structure like so:
|
25
|
+
#
|
26
|
+
# postgres/
|
27
|
+
# data/...
|
28
|
+
# bin/wal_shipper.rb
|
29
|
+
# etc/wal_shipper.conf <-- YAML settings!
|
30
|
+
# wal/
|
31
|
+
#
|
32
|
+
# It should be loaded from the PostgreSQL master's postgresql.conf
|
33
|
+
# as such, after putting it into your postgres user homedir under 'bin':
|
34
|
+
#
|
35
|
+
# archive_command = '/path/to/postgres_home/bin/wal_shipper.rb %p'
|
36
|
+
#
|
37
|
+
# Passwordless ssh keys need to be set up for the postgres user on all
|
38
|
+
# participating masters and slaves.
|
39
|
+
#
|
40
|
+
# You can use any replay method of your choosing on the slaves.
|
41
|
+
# Here's a nice example using pg_standby, to be put in data/recovery.conf:
|
42
|
+
#
|
43
|
+
# restore_command = 'pg_standby -t /tmp/pgrecovery.done -s5 -w0 -c /path/to/postgres_home/wal_files/ %f %p %r'
|
44
|
+
#
|
45
|
+
# Or, here's another simple alternative data/recovery.conf, for using WAL shipping
|
46
|
+
# alongside streaming replication:
|
47
|
+
#
|
48
|
+
# standby_mode = 'on'
|
49
|
+
# primary_conninfo = 'host=master.example.com port=5432 user=repl password=XXXXXXX'
|
50
|
+
# restore_command = 'cp /usr/local/pgsql/wal/%f %p'
|
51
|
+
# trigger_file = '/usr/local/pgsql/pg.become_primary'
|
52
|
+
# archive_cleanup_command = '/usr/local/bin/pg_archivecleanup /usr/local/pgsql/wal %r'
|
53
|
+
#
|
54
|
+
#========================================================================================
|
55
|
+
|
56
|
+
|
57
|
+
require 'pathname'
|
58
|
+
require 'yaml'
|
59
|
+
require 'fileutils'
|
60
|
+
require 'ostruct'
|
61
|
+
|
62
|
+
|
63
|
+
### Encapsulate WAL shipping functionality.
|
64
|
+
###
|
65
|
+
module WalShipper
|
66
|
+
|
67
|
+
### Send messages to the PostgreSQL log files.
|
68
|
+
###
|
69
|
+
def log( msg )
|
70
|
+
return unless @debug
|
71
|
+
puts "WAL Shipper: %s" % [ msg ]
|
72
|
+
end
|
73
|
+
|
74
|
+
|
75
|
+
### An object that represents a single destination from the
|
76
|
+
### configuration file.
|
77
|
+
###
|
78
|
+
class Destination < OpenStruct
|
79
|
+
include WalShipper
|
80
|
+
|
81
|
+
### Create a new WalShipper::Destination object.
|
82
|
+
def initialize( dest, debug=false )
|
83
|
+
@debug = debug
|
84
|
+
super( dest )
|
85
|
+
self.validate
|
86
|
+
end
|
87
|
+
|
88
|
+
#########
|
89
|
+
protected
|
90
|
+
#########
|
91
|
+
|
92
|
+
|
93
|
+
### Check for required keys and normalize various keys.
|
94
|
+
###
|
95
|
+
def validate
|
96
|
+
# Check for required destination keys
|
97
|
+
%w[ label kind ].each do |key|
|
98
|
+
if self.send( key.to_sym ).nil?
|
99
|
+
self.log "Destination %p missing required '%s' key." % [ self, key ]
|
100
|
+
self.invalid = true
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
# Ensure paths are Pathnames for the 'file' destination type.
|
105
|
+
self.path = Pathname.new( self.path ) if self.kind == 'file'
|
106
|
+
|
107
|
+
if self.kind == 'rsync-ssh'
|
108
|
+
self.port ||= 22
|
109
|
+
self.user = self.user ? "#{self.user}@" : ''
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end # Class Destination
|
113
|
+
|
114
|
+
|
115
|
+
|
116
|
+
### Class for creating new Destination objects and determining how to
|
117
|
+
### ship WAL files to them.
|
118
|
+
###
|
119
|
+
class Dispatcher
|
120
|
+
include WalShipper
|
121
|
+
|
122
|
+
### Create a new Shipper object, given a +conf+ hash and a +wal+ file
|
123
|
+
### Pathname object.
|
124
|
+
###
|
125
|
+
def initialize( wal, conf )
|
126
|
+
# Make the config keys instance variables.
|
127
|
+
conf.each_pair {|key, val| self.instance_variable_set( "@#{key}", val ) }
|
128
|
+
|
129
|
+
# Spool directory check.
|
130
|
+
#
|
131
|
+
@spool = Pathname.new( @spool )
|
132
|
+
@spool.exist? or raise "The configured spool directory (%s) doesn't exist." % [ @spool ]
|
133
|
+
|
134
|
+
# Stop right away if we have disabled shipping.
|
135
|
+
#
|
136
|
+
unless @enabled
|
137
|
+
self.log "WAL shipping is disabled, queuing segment %s" % [ wal.basename ]
|
138
|
+
exit 1
|
139
|
+
end
|
140
|
+
|
141
|
+
# Instantiate Destination objects, creating new spool directories
|
142
|
+
# for each.
|
143
|
+
#
|
144
|
+
@destinations.
|
145
|
+
collect!{|dest| WalShipper::Destination.new( dest, @debug ) }.
|
146
|
+
reject {|dest| dest.invalid }.
|
147
|
+
collect do |dest|
|
148
|
+
dest.spool = @spool + dest.label
|
149
|
+
dest.spool.mkdir( 0711 ) unless dest.spool.exist?
|
150
|
+
dest
|
151
|
+
end
|
152
|
+
|
153
|
+
# Put the WAL file into the spool for processing!
|
154
|
+
#
|
155
|
+
@waldir = @spool + 'wal_segments'
|
156
|
+
@waldir.mkdir( 0711 ) unless @waldir.exist?
|
157
|
+
|
158
|
+
self.log "Copying %s to %s" % [ wal.basename, @waldir ]
|
159
|
+
FileUtils::cp wal, @waldir
|
160
|
+
|
161
|
+
# 'wal' now references the copy. The original is managed and auto-expired
|
162
|
+
# by PostgreSQL when a new checkpoint segment it reached.
|
163
|
+
@wal = @waldir + wal.basename
|
164
|
+
end
|
165
|
+
|
166
|
+
|
167
|
+
### Create hardlinks for the WAL file into each of the destination directories
|
168
|
+
### for separate queueing and recording of what was shipped successfully.
|
169
|
+
###
|
170
|
+
def link
|
171
|
+
@destinations.each do |dest|
|
172
|
+
self.log "Linking %s into %s" % [ @wal.basename, dest.spool.basename ]
|
173
|
+
FileUtils::ln @wal, dest.spool, :force => true
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
|
178
|
+
### Decide to be synchronous or threaded, and delegate each destination
|
179
|
+
### to the proper ship method.
|
180
|
+
###
|
181
|
+
def dispatch
|
182
|
+
# Synchronous mode.
|
183
|
+
#
|
184
|
+
unless @async
|
185
|
+
self.log "Performing a synchronous dispatch."
|
186
|
+
@destinations.each {|dest| self.dispatch_dest( dest ) }
|
187
|
+
return
|
188
|
+
end
|
189
|
+
|
190
|
+
tg = ThreadGroup.new
|
191
|
+
|
192
|
+
# Async, one thread per destination
|
193
|
+
#
|
194
|
+
if @async_max.nil? || @async_max.to_i.zero?
|
195
|
+
self.log "Performing an asynchronous dispatch: one thread per destination."
|
196
|
+
@destinations.each do |dest|
|
197
|
+
t = Thread.new do
|
198
|
+
Thread.current.abort_on_exception = true
|
199
|
+
self.dispatch_dest( dest )
|
200
|
+
end
|
201
|
+
tg.add( t )
|
202
|
+
end
|
203
|
+
tg.list.each {|t| t.join }
|
204
|
+
return
|
205
|
+
end
|
206
|
+
|
207
|
+
# Async, one thread per destination, in groups of asynx_max size.
|
208
|
+
#
|
209
|
+
self.log "Performing an asynchronous dispatch: one thread per destination, %d at a time." % [ @async_max ]
|
210
|
+
all_dests = @destinations.dup
|
211
|
+
dest_chunks = []
|
212
|
+
until all_dests.empty? do
|
213
|
+
dest_chunks << all_dests.slice!( 0, @async_max )
|
214
|
+
end
|
215
|
+
|
216
|
+
dest_chunks.each do |chunk|
|
217
|
+
chunk.each do |dest|
|
218
|
+
t = Thread.new do
|
219
|
+
Thread.current.abort_on_exception = true
|
220
|
+
self.dispatch_dest( dest )
|
221
|
+
end
|
222
|
+
tg.add( t )
|
223
|
+
end
|
224
|
+
|
225
|
+
tg.list.each {|t| t.join }
|
226
|
+
end
|
227
|
+
|
228
|
+
return
|
229
|
+
end
|
230
|
+
|
231
|
+
|
232
|
+
### Remove any WAL segments no longer needed by slaves.
|
233
|
+
###
|
234
|
+
def clean_spool
|
235
|
+
total = 0
|
236
|
+
@waldir.children.each do |wal|
|
237
|
+
if wal.stat.nlink == 1
|
238
|
+
total += wal.unlink
|
239
|
+
end
|
240
|
+
end
|
241
|
+
|
242
|
+
self.log "Removed %d WAL segment%s." % [ total, total == 1 ? '' : 's' ]
|
243
|
+
end
|
244
|
+
|
245
|
+
|
246
|
+
|
247
|
+
#########
|
248
|
+
protected
|
249
|
+
#########
|
250
|
+
|
251
|
+
### Send WAL segments to remote +dest+ via rsync+ssh.
|
252
|
+
### Passwordless keys between the user running this script (postmaster owner)
|
253
|
+
### and remote user need to be set up in advance.
|
254
|
+
###
|
255
|
+
def ship_rsync_ssh( dest )
|
256
|
+
if dest.host.nil?
|
257
|
+
self.log "Destination %p missing required 'host' key. WAL is queued." % [ dest.host ]
|
258
|
+
return
|
259
|
+
end
|
260
|
+
|
261
|
+
rsync_flags = '-zc'
|
262
|
+
ssh_string = "%s -o ConnectTimeout=%d -o StrictHostKeyChecking=no -p %d" %
|
263
|
+
[ @ssh, @ssh_timeout || 10, dest.port ]
|
264
|
+
src_string = ''
|
265
|
+
dst_string = "%s%s:%s/" % [ dest.user, dest.host, dest.path ]
|
266
|
+
|
267
|
+
# If there are numerous files in the spool dir, it means there was
|
268
|
+
# an error transferring to this host in the past. Try and ship all
|
269
|
+
# WAL segments, instead of just the new one. PostgreSQL on the slave
|
270
|
+
# side will "do the right thing" as they come in, regardless of
|
271
|
+
# ordering.
|
272
|
+
#
|
273
|
+
if dest.spool.children.length > 1
|
274
|
+
src_string = dest.spool.to_s + '/'
|
275
|
+
rsync_flags << 'r'
|
276
|
+
else
|
277
|
+
src_string = dest.spool + @wal.basename
|
278
|
+
end
|
279
|
+
|
280
|
+
|
281
|
+
ship_wal_cmd = [
|
282
|
+
@rsync,
|
283
|
+
@debug ? (rsync_flags << 'vh') : (rsync_flags << 'q'),
|
284
|
+
'--remove-source-files',
|
285
|
+
'-e', ssh_string,
|
286
|
+
src_string, dst_string
|
287
|
+
]
|
288
|
+
|
289
|
+
self.log "Running command '%s'" % [ ship_wal_cmd.join(' ') ]
|
290
|
+
system *ship_wal_cmd
|
291
|
+
|
292
|
+
# Run external notification program on error, if one is configured.
|
293
|
+
#
|
294
|
+
unless $?.success?
|
295
|
+
self.log "Ack! Error while shipping to %p, WAL is queued." % [ dest.label ]
|
296
|
+
system @error_cmd, dest.label if @error_cmd
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
300
|
+
|
301
|
+
### Copy WAL segments to remote path as set in +dest+.
|
302
|
+
### This is useful for longer term PITR, copying to NFS shares, etc.
|
303
|
+
###
|
304
|
+
def ship_file( dest )
|
305
|
+
if dest.path.nil?
|
306
|
+
self.log "Destination %p missing required 'path' key. WAL is queued." % [ dest ]
|
307
|
+
return
|
308
|
+
end
|
309
|
+
dest.path.mkdir( 0711 ) unless dest.path.exist?
|
310
|
+
|
311
|
+
# If there are numerous files in the spool dir, it means there was
|
312
|
+
# an error transferring to this host in the past. Try and ship all
|
313
|
+
# WAL segments, instead of just the new one. PostgreSQL on the slave
|
314
|
+
# side will "do the right thing" as they come in, regardless of
|
315
|
+
# ordering.
|
316
|
+
#
|
317
|
+
if dest.spool.children.length > 1
|
318
|
+
dest.spool.children.each do |wal|
|
319
|
+
wal.unlink if self.copy_file( wal, dest.path, dest.label, dest.compress )
|
320
|
+
end
|
321
|
+
else
|
322
|
+
wal = dest.spool + @wal.basename
|
323
|
+
wal.unlink if self.copy_file( wal, dest.path, dest.label, dest.compress )
|
324
|
+
end
|
325
|
+
end
|
326
|
+
|
327
|
+
|
328
|
+
### Given a +wal+ Pathname, a +path+ destination, and the destination
|
329
|
+
### label, copy and optionally compress a WAL file.
|
330
|
+
###
|
331
|
+
def copy_file( wal, path, label, compress=false )
|
332
|
+
dest_file = path + wal.basename
|
333
|
+
FileUtils::cp wal, dest_file
|
334
|
+
if compress
|
335
|
+
system *[ 'gzip', '-f', dest_file ]
|
336
|
+
raise "Error while compressing: %s" % [ wal.basename ] unless $?.success?
|
337
|
+
end
|
338
|
+
self.log "Copied %s%s to %s." %
|
339
|
+
[ wal.basename, compress ? ' (and compressed)' : '', path ]
|
340
|
+
return true
|
341
|
+
rescue => err
|
342
|
+
self.log "Ack! Error while copying '%s' (%s) to %p, WAL is queued." %
|
343
|
+
[ wal.basename, err.message, path ]
|
344
|
+
system @error_cmd, label if @error_cmd
|
345
|
+
return false
|
346
|
+
end
|
347
|
+
|
348
|
+
|
349
|
+
### Figure out how to send the WAL file to its intended destination +dest+.
|
350
|
+
###
|
351
|
+
def dispatch_dest( dest )
|
352
|
+
if ! dest.enabled.nil? && ! dest.enabled
|
353
|
+
self.log "Skipping explicity disabled destination %p, WAL is queued." % [ dest.label ]
|
354
|
+
return
|
355
|
+
end
|
356
|
+
|
357
|
+
# Send to the appropriate method. ( rsync-ssh --> ship_rsync_ssh )
|
358
|
+
#
|
359
|
+
meth = ( 'ship_' + dest.kind.gsub(/-/, '_') ).to_sym
|
360
|
+
if WalShipper::Dispatcher.method_defined?( meth )
|
361
|
+
self.send( meth, dest )
|
362
|
+
else
|
363
|
+
self.log "Unknown destination kind %p for %p. WAL is queued." % [ dest.kind, dest.label ]
|
364
|
+
end
|
365
|
+
end
|
366
|
+
end
|
367
|
+
end
|
368
|
+
|
369
|
+
# Ship the WAL file!
|
370
|
+
#
|
371
|
+
if __FILE__ == $0
|
372
|
+
CONFIG_DIR = Pathname.new( __FILE__ ).dirname.parent + 'etc'
|
373
|
+
CONFIG = CONFIG_DIR + 'wal_shipper.conf'
|
374
|
+
|
375
|
+
unless CONFIG.exist?
|
376
|
+
CONFIG_DIR.mkdir( 0711 ) unless CONFIG_DIR.exist?
|
377
|
+
CONFIG.open('w') {|conf| conf.print(DATA.read) }
|
378
|
+
CONFIG.chmod( 0644 )
|
379
|
+
puts "No WAL shipping configuration found, default file created."
|
380
|
+
end
|
381
|
+
|
382
|
+
wal = ARGV[0] or raise "No WAL file was specified on the command line."
|
383
|
+
wal = Pathname.new( wal )
|
384
|
+
conf = YAML.load( CONFIG.read )
|
385
|
+
|
386
|
+
shipper = WalShipper::Dispatcher.new( wal, conf )
|
387
|
+
shipper.link
|
388
|
+
shipper.dispatch
|
389
|
+
shipper.clean_spool
|
390
|
+
end
|
391
|
+
|
392
|
+
|
393
|
+
__END__
|
394
|
+
---
|
395
|
+
# Spool from pg_xlog to the working area?
|
396
|
+
# This must be set to 'true' for wal shipping to function!
|
397
|
+
enabled: false
|
398
|
+
|
399
|
+
# Log everything to the PostgreSQL log files?
|
400
|
+
debug: true
|
401
|
+
|
402
|
+
# The working area for WAL segments.
|
403
|
+
spool: /opt/local/var/db/postgresql84/wal
|
404
|
+
|
405
|
+
# With multiple slaves, ship WAL in parallel, or be synchronous?
|
406
|
+
async: false
|
407
|
+
|
408
|
+
# Put a ceiling on the parallel threads?
|
409
|
+
# '0' or removing this option uses a thread for each destination,
|
410
|
+
# regardless of how many you have. Keep in mind that's 16 * destination
|
411
|
+
# count megs of simultaneous bandwidth.
|
412
|
+
async_max: 5
|
413
|
+
|
414
|
+
# Paths and settings for various binaries.
|
415
|
+
rsync: /usr/bin/rsync
|
416
|
+
ssh: /usr/bin/ssh
|
417
|
+
ssh_timeout: 10
|
418
|
+
|
419
|
+
destinations:
|
420
|
+
|
421
|
+
- label: rsync-example
|
422
|
+
port: 2222
|
423
|
+
kind: rsync-ssh
|
424
|
+
host: localhost
|
425
|
+
user: postgres
|
426
|
+
path: wal # relative to the user's homedir on the remote host
|
427
|
+
enabled: false
|
428
|
+
|
429
|
+
- label: file-example
|
430
|
+
kind: file
|
431
|
+
compress: true
|
432
|
+
enabled: true
|
433
|
+
path: /tmp/someplace
|
434
|
+
|