cipherstash-pg 1.0.0.beta.1-aarch64-linux → 1.0.0.beta.4-aarch64-linux

Sign up to get free protection for your applications and to get access to all the features.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/.appveyor.yml +42 -0
  3. data/.gems +6 -0
  4. data/.gemtest +0 -0
  5. data/.github/workflows/binary-gems.yml +117 -0
  6. data/.github/workflows/source-gem.yml +137 -0
  7. data/.gitignore +19 -0
  8. data/.hgsigs +34 -0
  9. data/.hgtags +41 -0
  10. data/.irbrc +23 -0
  11. data/.pryrc +23 -0
  12. data/.tm_properties +21 -0
  13. data/.travis.yml +49 -0
  14. data/Gemfile +3 -3
  15. data/Gemfile.lock +45 -0
  16. data/{History.rdoc → History.md} +168 -153
  17. data/README.ja.md +266 -0
  18. data/README.md +272 -0
  19. data/Rakefile +65 -104
  20. data/Rakefile.cross +298 -0
  21. data/certs/larskanis-2023.pem +24 -0
  22. data/cipherstash-pg.gemspec +0 -0
  23. data/lib/2.7/pg_ext.so +0 -0
  24. data/lib/3.0/pg_ext.so +0 -0
  25. data/lib/3.1/pg_ext.so +0 -0
  26. data/lib/3.2/pg_ext.so +0 -0
  27. data/lib/cipherstash-pg/basic_type_map_based_on_result.rb +11 -0
  28. data/lib/cipherstash-pg/basic_type_map_for_queries.rb +113 -0
  29. data/lib/cipherstash-pg/basic_type_map_for_results.rb +30 -0
  30. data/lib/cipherstash-pg/basic_type_registry.rb +206 -0
  31. data/lib/cipherstash-pg/binary_decoder.rb +21 -0
  32. data/lib/cipherstash-pg/coder.rb +82 -0
  33. data/lib/cipherstash-pg/connection.rb +467 -0
  34. data/lib/cipherstash-pg/constants.rb +3 -0
  35. data/lib/cipherstash-pg/exceptions.rb +19 -0
  36. data/lib/cipherstash-pg/result.rb +22 -0
  37. data/lib/cipherstash-pg/text_decoder.rb +43 -0
  38. data/lib/cipherstash-pg/text_encoder.rb +67 -0
  39. data/lib/cipherstash-pg/tuple.rb +24 -0
  40. data/lib/cipherstash-pg/type_map_by_column.rb +11 -0
  41. data/lib/cipherstash-pg/version.rb +3 -0
  42. data/lib/cipherstash-pg.rb +56 -11
  43. data/lib/libpq.so.5 +0 -0
  44. data/misc/openssl-pg-segfault.rb +15 -25
  45. data/misc/postgres/Rakefile +13 -20
  46. data/misc/postgres/lib/postgres.rb +10 -14
  47. data/misc/ruby-pg/Rakefile +13 -20
  48. data/misc/ruby-pg/lib/ruby/pg.rb +10 -14
  49. data/rakelib/task_extension.rb +17 -31
  50. data/sample/array_insert.rb +7 -20
  51. data/sample/async_api.rb +54 -96
  52. data/sample/async_copyto.rb +20 -35
  53. data/sample/async_mixed.rb +22 -50
  54. data/sample/check_conn.rb +8 -20
  55. data/sample/copydata.rb +18 -68
  56. data/sample/copyfrom.rb +26 -78
  57. data/sample/copyto.rb +10 -16
  58. data/sample/cursor.rb +9 -19
  59. data/sample/disk_usage_report.rb +89 -174
  60. data/sample/issue-119.rb +45 -93
  61. data/sample/losample.rb +48 -66
  62. data/sample/minimal-testcase.rb +6 -17
  63. data/sample/notify_wait.rb +21 -67
  64. data/sample/pg_statistics.rb +100 -281
  65. data/sample/replication_monitor.rb +119 -218
  66. data/sample/test_binary_values.rb +14 -30
  67. data/sample/wal_shipper.rb +199 -431
  68. data/sample/warehouse_partitions.rb +157 -307
  69. data/translation/.po4a-version +7 -0
  70. data/translation/po/all.pot +875 -0
  71. data/translation/po/ja.po +868 -0
  72. data/translation/po4a.cfg +9 -0
  73. metadata +50 -28
  74. data/README.ja.rdoc +0 -13
  75. data/README.rdoc +0 -233
  76. data/lib/pg/basic_type_map_based_on_result.rb +0 -47
  77. data/lib/pg/basic_type_map_for_queries.rb +0 -193
  78. data/lib/pg/basic_type_map_for_results.rb +0 -81
  79. data/lib/pg/basic_type_registry.rb +0 -301
  80. data/lib/pg/binary_decoder.rb +0 -23
  81. data/lib/pg/coder.rb +0 -104
  82. data/lib/pg/connection.rb +0 -878
  83. data/lib/pg/constants.rb +0 -12
  84. data/lib/pg/exceptions.rb +0 -18
  85. data/lib/pg/result.rb +0 -43
  86. data/lib/pg/text_decoder.rb +0 -46
  87. data/lib/pg/text_encoder.rb +0 -59
  88. data/lib/pg/tuple.rb +0 -30
  89. data/lib/pg/type_map_by_column.rb +0 -16
  90. data/lib/pg/version.rb +0 -4
  91. data/lib/pg.rb +0 -55
@@ -1,434 +1,202 @@
1
- # -*- ruby -*-
2
- #
3
- # A script to wrap ssh and rsync for PostgreSQL WAL files shipping.
4
- # Mahlon E. Smith <mahlon@martini.nu>
5
- #
6
- # Based off of Joshua Drake's PITRTools concept, but with some important
7
- # differences:
8
- #
9
- # - Only supports PostgreSQL >= 8.3
10
- # - No support for rsync version < 3
11
- # - Only shipping, no client side sync (too much opportunity for failure,
12
- # and it's easy to get a base backup manually)
13
- # - WAL files are only stored once, regardless of how many
14
- # slaves are configured or not responding, and are removed from
15
- # the master when they are no longer needed.
16
- # - Each slave can have completely distinct settings, instead
17
- # of a single set of options applied to all slaves
18
- # - slave sync can be individually paused from the master
19
- # - can run synchronously, or if you have a lot of slaves, threaded async mode
20
- # - It's ruby, instead of python. :)
21
- #
22
- # wal_shipper is configurable via an external YAML file, and will create
23
- # a template on its first run -- you'll need to modify it! It expects
24
- # a directory structure like so:
25
- #
26
- # postgres/
27
- # data/...
28
- # bin/wal_shipper.rb
29
- # etc/wal_shipper.conf <-- YAML settings!
30
- # wal/
31
- #
32
- # It should be loaded from the PostgreSQL master's postgresql.conf
33
- # as such, after putting it into your postgres user homedir under 'bin':
34
- #
35
- # archive_command = '/path/to/postgres_home/bin/wal_shipper.rb %p'
36
- #
37
- # Passwordless ssh keys need to be set up for the postgres user on all
38
- # participating masters and slaves.
39
- #
40
- # You can use any replay method of your choosing on the slaves.
41
- # Here's a nice example using pg_standby, to be put in data/recovery.conf:
42
- #
43
- # restore_command = 'pg_standby -t /tmp/pgrecovery.done -s5 -w0 -c /path/to/postgres_home/wal_files/ %f %p %r'
44
- #
45
- # Or, here's another simple alternative data/recovery.conf, for using WAL shipping
46
- # alongside streaming replication:
47
- #
48
- # standby_mode = 'on'
49
- # primary_conninfo = 'host=master.example.com port=5432 user=repl password=XXXXXXX'
50
- # restore_command = 'cp /usr/local/pgsql/wal/%f %p'
51
- # trigger_file = '/usr/local/pgsql/pg.become_primary'
52
- # archive_cleanup_command = '/usr/local/bin/pg_archivecleanup /usr/local/pgsql/wal %r'
53
- #
54
- #========================================================================================
55
-
56
-
57
- require 'pathname'
58
- require 'yaml'
59
- require 'fileutils'
60
- require 'ostruct'
61
-
62
-
63
- ### Encapsulate WAL shipping functionality.
64
- ###
1
+ require("pathname")
2
+ require("yaml")
3
+ require("fileutils")
4
+ require("ostruct")
65
5
  module WalShipper
66
-
67
- ### Send messages to the PostgreSQL log files.
68
- ###
69
- def log( msg )
70
- return unless @debug
71
- puts "WAL Shipper: %s" % [ msg ]
72
- end
73
-
74
-
75
- ### An object that represents a single destination from the
76
- ### configuration file.
77
- ###
78
- class Destination < OpenStruct
79
- include WalShipper
80
-
81
- ### Create a new WalShipper::Destination object.
82
- def initialize( dest, debug=false )
83
- @debug = debug
84
- super( dest )
85
- self.validate
86
- end
87
-
88
- #########
89
- protected
90
- #########
91
-
92
-
93
- ### Check for required keys and normalize various keys.
94
- ###
95
- def validate
96
- # Check for required destination keys
97
- %w[ label kind ].each do |key|
98
- if self.send( key.to_sym ).nil?
99
- self.log "Destination %p missing required '%s' key." % [ self, key ]
100
- self.invalid = true
101
- end
102
- end
103
-
104
- # Ensure paths are Pathnames for the 'file' destination type.
105
- self.path = Pathname.new( self.path ) if self.kind == 'file'
106
-
107
- if self.kind == 'rsync-ssh'
108
- self.port ||= 22
109
- self.user = self.user ? "#{self.user}@" : ''
110
- end
111
- end
112
- end # Class Destination
113
-
114
-
115
-
116
- ### Class for creating new Destination objects and determining how to
117
- ### ship WAL files to them.
118
- ###
119
- class Dispatcher
120
- include WalShipper
121
-
122
- ### Create a new Shipper object, given a +conf+ hash and a +wal+ file
123
- ### Pathname object.
124
- ###
125
- def initialize( wal, conf )
126
- # Make the config keys instance variables.
127
- conf.each_pair {|key, val| self.instance_variable_set( "@#{key}", val ) }
128
-
129
- # Spool directory check.
130
- #
131
- @spool = Pathname.new( @spool )
132
- @spool.exist? or raise "The configured spool directory (%s) doesn't exist." % [ @spool ]
133
-
134
- # Stop right away if we have disabled shipping.
135
- #
136
- unless @enabled
137
- self.log "WAL shipping is disabled, queuing segment %s" % [ wal.basename ]
138
- exit 1
139
- end
140
-
141
- # Instantiate Destination objects, creating new spool directories
142
- # for each.
143
- #
144
- @destinations.
145
- collect!{|dest| WalShipper::Destination.new( dest, @debug ) }.
146
- reject {|dest| dest.invalid }.
147
- collect do |dest|
148
- dest.spool = @spool + dest.label
149
- dest.spool.mkdir( 0711 ) unless dest.spool.exist?
150
- dest
151
- end
152
-
153
- # Put the WAL file into the spool for processing!
154
- #
155
- @waldir = @spool + 'wal_segments'
156
- @waldir.mkdir( 0711 ) unless @waldir.exist?
157
-
158
- self.log "Copying %s to %s" % [ wal.basename, @waldir ]
159
- FileUtils::cp wal, @waldir
160
-
161
- # 'wal' now references the copy. The original is managed and auto-expired
162
- # by PostgreSQL when a new checkpoint segment it reached.
163
- @wal = @waldir + wal.basename
164
- end
165
-
166
-
167
- ### Create hardlinks for the WAL file into each of the destination directories
168
- ### for separate queueing and recording of what was shipped successfully.
169
- ###
170
- def link
171
- @destinations.each do |dest|
172
- self.log "Linking %s into %s" % [ @wal.basename, dest.spool.basename ]
173
- FileUtils::ln @wal, dest.spool, :force => true
174
- end
175
- end
176
-
177
-
178
- ### Decide to be synchronous or threaded, and delegate each destination
179
- ### to the proper ship method.
180
- ###
181
- def dispatch
182
- # Synchronous mode.
183
- #
184
- unless @async
185
- self.log "Performing a synchronous dispatch."
186
- @destinations.each {|dest| self.dispatch_dest( dest ) }
187
- return
188
- end
189
-
190
- tg = ThreadGroup.new
191
-
192
- # Async, one thread per destination
193
- #
194
- if @async_max.nil? || @async_max.to_i.zero?
195
- self.log "Performing an asynchronous dispatch: one thread per destination."
196
- @destinations.each do |dest|
197
- t = Thread.new do
198
- Thread.current.abort_on_exception = true
199
- self.dispatch_dest( dest )
200
- end
201
- tg.add( t )
202
- end
203
- tg.list.each {|t| t.join }
204
- return
205
- end
206
-
207
- # Async, one thread per destination, in groups of asynx_max size.
208
- #
209
- self.log "Performing an asynchronous dispatch: one thread per destination, %d at a time." % [ @async_max ]
210
- all_dests = @destinations.dup
211
- dest_chunks = []
212
- until all_dests.empty? do
213
- dest_chunks << all_dests.slice!( 0, @async_max )
214
- end
215
-
216
- dest_chunks.each do |chunk|
217
- chunk.each do |dest|
218
- t = Thread.new do
219
- Thread.current.abort_on_exception = true
220
- self.dispatch_dest( dest )
221
- end
222
- tg.add( t )
223
- end
224
-
225
- tg.list.each {|t| t.join }
226
- end
227
-
228
- return
229
- end
230
-
231
-
232
- ### Remove any WAL segments no longer needed by slaves.
233
- ###
234
- def clean_spool
235
- total = 0
236
- @waldir.children.each do |wal|
237
- if wal.stat.nlink == 1
238
- total += wal.unlink
239
- end
240
- end
241
-
242
- self.log "Removed %d WAL segment%s." % [ total, total == 1 ? '' : 's' ]
243
- end
244
-
245
-
246
-
247
- #########
248
- protected
249
- #########
250
-
251
- ### Send WAL segments to remote +dest+ via rsync+ssh.
252
- ### Passwordless keys between the user running this script (postmaster owner)
253
- ### and remote user need to be set up in advance.
254
- ###
255
- def ship_rsync_ssh( dest )
256
- if dest.host.nil?
257
- self.log "Destination %p missing required 'host' key. WAL is queued." % [ dest.host ]
258
- return
259
- end
260
-
261
- rsync_flags = '-zc'
262
- ssh_string = "%s -o ConnectTimeout=%d -o StrictHostKeyChecking=no -p %d" %
263
- [ @ssh, @ssh_timeout || 10, dest.port ]
264
- src_string = ''
265
- dst_string = "%s%s:%s/" % [ dest.user, dest.host, dest.path ]
266
-
267
- # If there are numerous files in the spool dir, it means there was
268
- # an error transferring to this host in the past. Try and ship all
269
- # WAL segments, instead of just the new one. PostgreSQL on the slave
270
- # side will "do the right thing" as they come in, regardless of
271
- # ordering.
272
- #
273
- if dest.spool.children.length > 1
274
- src_string = dest.spool.to_s + '/'
275
- rsync_flags << 'r'
276
- else
277
- src_string = dest.spool + @wal.basename
278
- end
279
-
280
-
281
- ship_wal_cmd = [
282
- @rsync,
283
- @debug ? (rsync_flags << 'vh') : (rsync_flags << 'q'),
284
- '--remove-source-files',
285
- '-e', ssh_string,
286
- src_string, dst_string
287
- ]
288
-
289
- self.log "Running command '%s'" % [ ship_wal_cmd.join(' ') ]
290
- system *ship_wal_cmd
291
-
292
- # Run external notification program on error, if one is configured.
293
- #
294
- unless $?.success?
295
- self.log "Ack! Error while shipping to %p, WAL is queued." % [ dest.label ]
296
- system @error_cmd, dest.label if @error_cmd
297
- end
298
- end
299
-
300
-
301
- ### Copy WAL segments to remote path as set in +dest+.
302
- ### This is useful for longer term PITR, copying to NFS shares, etc.
303
- ###
304
- def ship_file( dest )
305
- if dest.path.nil?
306
- self.log "Destination %p missing required 'path' key. WAL is queued." % [ dest ]
307
- return
308
- end
309
- dest.path.mkdir( 0711 ) unless dest.path.exist?
310
-
311
- # If there are numerous files in the spool dir, it means there was
312
- # an error transferring to this host in the past. Try and ship all
313
- # WAL segments, instead of just the new one. PostgreSQL on the slave
314
- # side will "do the right thing" as they come in, regardless of
315
- # ordering.
316
- #
317
- if dest.spool.children.length > 1
318
- dest.spool.children.each do |wal|
319
- wal.unlink if self.copy_file( wal, dest.path, dest.label, dest.compress )
320
- end
321
- else
322
- wal = dest.spool + @wal.basename
323
- wal.unlink if self.copy_file( wal, dest.path, dest.label, dest.compress )
324
- end
325
- end
326
-
327
-
328
- ### Given a +wal+ Pathname, a +path+ destination, and the destination
329
- ### label, copy and optionally compress a WAL file.
330
- ###
331
- def copy_file( wal, path, label, compress=false )
332
- dest_file = path + wal.basename
333
- FileUtils::cp wal, dest_file
334
- if compress
335
- system *[ 'gzip', '-f', dest_file ]
336
- raise "Error while compressing: %s" % [ wal.basename ] unless $?.success?
337
- end
338
- self.log "Copied %s%s to %s." %
339
- [ wal.basename, compress ? ' (and compressed)' : '', path ]
340
- return true
341
- rescue => err
342
- self.log "Ack! Error while copying '%s' (%s) to %p, WAL is queued." %
343
- [ wal.basename, err.message, path ]
344
- system @error_cmd, label if @error_cmd
345
- return false
346
- end
347
-
348
-
349
- ### Figure out how to send the WAL file to its intended destination +dest+.
350
- ###
351
- def dispatch_dest( dest )
352
- if ! dest.enabled.nil? && ! dest.enabled
353
- self.log "Skipping explicitly disabled destination %p, WAL is queued." % [ dest.label ]
354
- return
355
- end
356
-
357
- # Send to the appropriate method. ( rsync-ssh --> ship_rsync_ssh )
358
- #
359
- meth = ( 'ship_' + dest.kind.gsub(/-/, '_') ).to_sym
360
- if WalShipper::Dispatcher.method_defined?( meth )
361
- self.send( meth, dest )
362
- else
363
- self.log "Unknown destination kind %p for %p. WAL is queued." % [ dest.kind, dest.label ]
364
- end
365
- end
366
- end
6
+ def log(msg)
7
+ return unless @debug
8
+ puts(("WAL Shipper: %s" % [msg]))
9
+ end
10
+
11
+ class Destination < OpenStruct
12
+ include(WalShipper)
13
+
14
+ def initialize(dest, debug = false)
15
+ @debug = debug
16
+ super(dest)
17
+ self.validate
18
+ end
19
+
20
+ protected
21
+
22
+ def validate
23
+ ["label", "kind"].each do |key|
24
+ if self.send(key.to_sym).nil? then
25
+ self.log(("Destination %p missing required '%s' key." % [self, key]))
26
+ self.invalid = true
27
+ end
28
+ end
29
+ self.path = Pathname.new(self.path) if (self.kind == "file")
30
+ if (self.kind == "rsync-ssh") then
31
+ self.port ||= 22
32
+ self.user = self.user ? ("#{self.user}@") : ("")
33
+ end
34
+ end
35
+ end
36
+
37
+ class Dispatcher
38
+ include(WalShipper)
39
+
40
+ def initialize(wal, conf)
41
+ conf.each_pair { |key, val| self.instance_variable_set("@#{key}", val) }
42
+ @spool = Pathname.new(@spool)
43
+ (@spool.exist? or raise(("The configured spool directory (%s) doesn't exist." % [@spool])))
44
+ unless @enabled then
45
+ self.log(("WAL shipping is disabled, queuing segment %s" % [wal.basename]))
46
+ exit(1)
47
+ end
48
+ @destinations.collect! { |dest| WalShipper::Destination.new(dest, @debug) }.reject do |dest|
49
+ dest.invalid
50
+ end.collect do |dest|
51
+ dest.spool = (@spool + dest.label)
52
+ dest.spool.mkdir(457) unless dest.spool.exist?
53
+ dest
54
+ end
55
+ @waldir = (@spool + "wal_segments")
56
+ @waldir.mkdir(457) unless @waldir.exist?
57
+ self.log(("Copying %s to %s" % [wal.basename, @waldir]))
58
+ FileUtils.cp(wal, @waldir)
59
+ @wal = (@waldir + wal.basename)
60
+ end
61
+
62
+ def link
63
+ @destinations.each do |dest|
64
+ self.log(("Linking %s into %s" % [@wal.basename, dest.spool.basename]))
65
+ FileUtils.ln(@wal, dest.spool, :force => true)
66
+ end
67
+ end
68
+
69
+ def dispatch
70
+ unless @async then
71
+ self.log("Performing a synchronous dispatch.")
72
+ @destinations.each { |dest| self.dispatch_dest(dest) }
73
+ return
74
+ end
75
+ tg = ThreadGroup.new
76
+ if (@async_max.nil? or @async_max.to_i.zero?) then
77
+ self.log("Performing an asynchronous dispatch: one thread per destination.")
78
+ @destinations.each do |dest|
79
+ t = Thread.new do
80
+ Thread.current.abort_on_exception = true
81
+ self.dispatch_dest(dest)
82
+ end
83
+ tg.add(t)
84
+ end
85
+ tg.list.each { |t| t.join }
86
+ return
87
+ end
88
+ self.log(("Performing an asynchronous dispatch: one thread per destination, %d at a time." % [@async_max]))
89
+ all_dests = @destinations.dup
90
+ dest_chunks = []
91
+ until all_dests.empty? do
92
+ (dest_chunks << all_dests.slice!(0, @async_max))
93
+ end
94
+ dest_chunks.each do |chunk|
95
+ chunk.each do |dest|
96
+ t = Thread.new do
97
+ Thread.current.abort_on_exception = true
98
+ self.dispatch_dest(dest)
99
+ end
100
+ tg.add(t)
101
+ end
102
+ tg.list.each { |t| t.join }
103
+ end
104
+ return
105
+ end
106
+
107
+ def clean_spool
108
+ total = 0
109
+ @waldir.children.each do |wal|
110
+ total = (total + wal.unlink) if (wal.stat.nlink == 1)
111
+ end
112
+ self.log(("Removed %d WAL segment%s." % [total, (total == 1) ? ("") : ("s")]))
113
+ end
114
+
115
+ protected
116
+
117
+ def ship_rsync_ssh(dest)
118
+ if dest.host.nil? then
119
+ self.log(("Destination %p missing required 'host' key. WAL is queued." % [dest.host]))
120
+ return
121
+ end
122
+ rsync_flags = "-zc"
123
+ ssh_string = ("%s -o ConnectTimeout=%d -o StrictHostKeyChecking=no -p %d" % [@ssh, (@ssh_timeout or 10), dest.port])
124
+ src_string = ""
125
+ dst_string = ("%s%s:%s/" % [dest.user, dest.host, dest.path])
126
+ if (dest.spool.children.length > 1) then
127
+ src_string = (dest.spool.to_s + "/")
128
+ (rsync_flags << "r")
129
+ else
130
+ src_string = (dest.spool + @wal.basename)
131
+ end
132
+ ship_wal_cmd = [@rsync, @debug ? ((rsync_flags << "vh")) : ((rsync_flags << "q")), "--remove-source-files", "-e", ssh_string, src_string, dst_string]
133
+ self.log(("Running command '%s'" % [ship_wal_cmd.join(" ")]))
134
+ system(*ship_wal_cmd)
135
+ unless $?.success? then
136
+ self.log(("Ack! Error while shipping to %p, WAL is queued." % [dest.label]))
137
+ system(@error_cmd, dest.label) if @error_cmd
138
+ end
139
+ end
140
+
141
+ def ship_file(dest)
142
+ if dest.path.nil? then
143
+ self.log(("Destination %p missing required 'path' key. WAL is queued." % [dest]))
144
+ return
145
+ end
146
+ dest.path.mkdir(457) unless dest.path.exist?
147
+ if (dest.spool.children.length > 1) then
148
+ dest.spool.children.each do |wal|
149
+ wal.unlink if self.copy_file(wal, dest.path, dest.label, dest.compress)
150
+ end
151
+ else
152
+ wal = (dest.spool + @wal.basename)
153
+ wal.unlink if self.copy_file(wal, dest.path, dest.label, dest.compress)
154
+ end
155
+ end
156
+
157
+ def copy_file(wal, path, label, compress = false)
158
+ (dest_file = (path + wal.basename)
159
+ FileUtils.cp(wal, dest_file)
160
+ if compress then
161
+ system(*["gzip", "-f", dest_file])
162
+ raise(("Error while compressing: %s" % [wal.basename])) unless $?.success?
163
+ end
164
+ self.log(("Copied %s%s to %s." % [wal.basename, compress ? (" (and compressed)") : (""), path]))
165
+ return true)
166
+ rescue => err
167
+ self.log(("Ack! Error while copying '%s' (%s) to %p, WAL is queued." % [wal.basename, err.message, path]))
168
+ system(@error_cmd, label) if @error_cmd
169
+ return false
170
+ end
171
+
172
+ def dispatch_dest(dest)
173
+ if (not dest.enabled.nil?) and (not dest.enabled) then
174
+ self.log(("Skipping explicitly disabled destination %p, WAL is queued." % [dest.label]))
175
+ return
176
+ end
177
+ meth = ("ship_" + dest.kind.gsub(/-/, "_")).to_sym
178
+ if WalShipper::Dispatcher.method_defined?(meth) then
179
+ self.send(meth, dest)
180
+ else
181
+ self.log(("Unknown destination kind %p for %p. WAL is queued." % [dest.kind, dest.label]))
182
+ end
183
+ end
184
+ end
367
185
  end
368
-
369
- # Ship the WAL file!
370
- #
371
- if __FILE__ == $0
372
- CONFIG_DIR = Pathname.new( __FILE__ ).dirname.parent + 'etc'
373
- CONFIG = CONFIG_DIR + 'wal_shipper.conf'
374
-
375
- unless CONFIG.exist?
376
- CONFIG_DIR.mkdir( 0711 ) unless CONFIG_DIR.exist?
377
- CONFIG.open('w') {|conf| conf.print(DATA.read) }
378
- CONFIG.chmod( 0644 )
379
- puts "No WAL shipping configuration found, default file created."
380
- end
381
-
382
- wal = ARGV[0] or raise "No WAL file was specified on the command line."
383
- wal = Pathname.new( wal )
384
- conf = YAML.load( CONFIG.read )
385
-
386
- shipper = WalShipper::Dispatcher.new( wal, conf )
387
- shipper.link
388
- shipper.dispatch
389
- shipper.clean_spool
186
+ if ("(string)" == $0) then
187
+ CONFIG_DIR = (Pathname.new("(string)").dirname.parent + "etc")
188
+ CONFIG = (CONFIG_DIR + "wal_shipper.conf")
189
+ unless CONFIG.exist? then
190
+ CONFIG_DIR.mkdir(457) unless CONFIG_DIR.exist?
191
+ CONFIG.open("w") { |conf| conf.print(DATA.read) }
192
+ CONFIG.chmod(420)
193
+ puts("No WAL shipping configuration found, default file created.")
194
+ end
195
+ (wal = ARGV[0] or raise("No WAL file was specified on the command line."))
196
+ wal = Pathname.new(wal)
197
+ conf = YAML.load(CONFIG.read)
198
+ shipper = WalShipper::Dispatcher.new(wal, conf)
199
+ shipper.link
200
+ shipper.dispatch
201
+ shipper.clean_spool
390
202
  end
391
-
392
-
393
- __END__
394
- ---
395
- # Spool from pg_xlog to the working area?
396
- # This must be set to 'true' for wal shipping to function!
397
- enabled: false
398
-
399
- # Log everything to the PostgreSQL log files?
400
- debug: true
401
-
402
- # The working area for WAL segments.
403
- spool: /opt/local/var/db/postgresql84/wal
404
-
405
- # With multiple slaves, ship WAL in parallel, or be synchronous?
406
- async: false
407
-
408
- # Put a ceiling on the parallel threads?
409
- # '0' or removing this option uses a thread for each destination,
410
- # regardless of how many you have. Keep in mind that's 16 * destination
411
- # count megs of simultaneous bandwidth.
412
- async_max: 5
413
-
414
- # Paths and settings for various binaries.
415
- rsync: /usr/bin/rsync
416
- ssh: /usr/bin/ssh
417
- ssh_timeout: 10
418
-
419
- destinations:
420
-
421
- - label: rsync-example
422
- port: 2222
423
- kind: rsync-ssh
424
- host: localhost
425
- user: postgres
426
- path: wal # relative to the user's homedir on the remote host
427
- enabled: false
428
-
429
- - label: file-example
430
- kind: file
431
- compress: true
432
- enabled: true
433
- path: /tmp/someplace
434
-