cipherstash-pg 1.0.0.beta.1-arm64-darwin-22

Sign up to get free protection for your applications and to get access to all the features.
Files changed (72) hide show
  1. checksums.yaml +7 -0
  2. data/BSDL +22 -0
  3. data/Contributors.rdoc +46 -0
  4. data/Gemfile +14 -0
  5. data/History.rdoc +789 -0
  6. data/LICENSE +56 -0
  7. data/Manifest.txt +72 -0
  8. data/POSTGRES +23 -0
  9. data/README-OS_X.rdoc +68 -0
  10. data/README-Windows.rdoc +56 -0
  11. data/README.ja.rdoc +13 -0
  12. data/README.rdoc +233 -0
  13. data/Rakefile +115 -0
  14. data/certs/ged.pem +24 -0
  15. data/certs/larskanis-2022.pem +26 -0
  16. data/cipherstash-pg.gemspec +31 -0
  17. data/lib/2.7/pg_ext.bundle +0 -0
  18. data/lib/3.0/pg_ext.bundle +0 -0
  19. data/lib/3.1/pg_ext.bundle +0 -0
  20. data/lib/3.2/pg_ext.bundle +0 -0
  21. data/lib/cipherstash-pg.rb +15 -0
  22. data/lib/libpq.5.dylib +0 -0
  23. data/lib/pg/basic_type_map_based_on_result.rb +47 -0
  24. data/lib/pg/basic_type_map_for_queries.rb +193 -0
  25. data/lib/pg/basic_type_map_for_results.rb +81 -0
  26. data/lib/pg/basic_type_registry.rb +301 -0
  27. data/lib/pg/binary_decoder.rb +23 -0
  28. data/lib/pg/coder.rb +104 -0
  29. data/lib/pg/connection.rb +878 -0
  30. data/lib/pg/constants.rb +12 -0
  31. data/lib/pg/exceptions.rb +18 -0
  32. data/lib/pg/result.rb +43 -0
  33. data/lib/pg/text_decoder.rb +46 -0
  34. data/lib/pg/text_encoder.rb +59 -0
  35. data/lib/pg/tuple.rb +30 -0
  36. data/lib/pg/type_map_by_column.rb +16 -0
  37. data/lib/pg/version.rb +4 -0
  38. data/lib/pg.rb +55 -0
  39. data/misc/openssl-pg-segfault.rb +31 -0
  40. data/misc/postgres/History.txt +9 -0
  41. data/misc/postgres/Manifest.txt +5 -0
  42. data/misc/postgres/README.txt +21 -0
  43. data/misc/postgres/Rakefile +21 -0
  44. data/misc/postgres/lib/postgres.rb +16 -0
  45. data/misc/ruby-pg/History.txt +9 -0
  46. data/misc/ruby-pg/Manifest.txt +5 -0
  47. data/misc/ruby-pg/README.txt +21 -0
  48. data/misc/ruby-pg/Rakefile +21 -0
  49. data/misc/ruby-pg/lib/ruby/pg.rb +16 -0
  50. data/rakelib/task_extension.rb +46 -0
  51. data/sample/array_insert.rb +20 -0
  52. data/sample/async_api.rb +102 -0
  53. data/sample/async_copyto.rb +39 -0
  54. data/sample/async_mixed.rb +56 -0
  55. data/sample/check_conn.rb +21 -0
  56. data/sample/copydata.rb +71 -0
  57. data/sample/copyfrom.rb +81 -0
  58. data/sample/copyto.rb +19 -0
  59. data/sample/cursor.rb +21 -0
  60. data/sample/disk_usage_report.rb +177 -0
  61. data/sample/issue-119.rb +94 -0
  62. data/sample/losample.rb +69 -0
  63. data/sample/minimal-testcase.rb +17 -0
  64. data/sample/notify_wait.rb +72 -0
  65. data/sample/pg_statistics.rb +285 -0
  66. data/sample/replication_monitor.rb +222 -0
  67. data/sample/test_binary_values.rb +33 -0
  68. data/sample/wal_shipper.rb +434 -0
  69. data/sample/warehouse_partitions.rb +311 -0
  70. data/vendor/database-extensions/install.sql +317 -0
  71. data/vendor/database-extensions/uninstall.sql +20 -0
  72. metadata +118 -0
@@ -0,0 +1,434 @@
1
+ # -*- ruby -*-
2
+ #
3
+ # A script to wrap ssh and rsync for PostgreSQL WAL files shipping.
4
+ # Mahlon E. Smith <mahlon@martini.nu>
5
+ #
6
+ # Based off of Joshua Drake's PITRTools concept, but with some important
7
+ # differences:
8
+ #
9
+ # - Only supports PostgreSQL >= 8.3
10
+ # - No support for rsync version < 3
11
+ # - Only shipping, no client side sync (too much opportunity for failure,
12
+ # and it's easy to get a base backup manually)
13
+ # - WAL files are only stored once, regardless of how many
14
+ # slaves are configured or not responding, and are removed from
15
+ # the master when they are no longer needed.
16
+ # - Each slave can have completely distinct settings, instead
17
+ # of a single set of options applied to all slaves
18
+ # - slave sync can be individually paused from the master
19
+ # - can run synchronously, or if you have a lot of slaves, threaded async mode
20
+ # - It's ruby, instead of python. :)
21
+ #
22
+ # wal_shipper is configurable via an external YAML file, and will create
23
+ # a template on its first run -- you'll need to modify it! It expects
24
+ # a directory structure like so:
25
+ #
26
+ # postgres/
27
+ # data/...
28
+ # bin/wal_shipper.rb
29
+ # etc/wal_shipper.conf <-- YAML settings!
30
+ # wal/
31
+ #
32
+ # It should be loaded from the PostgreSQL master's postgresql.conf
33
+ # as such, after putting it into your postgres user homedir under 'bin':
34
+ #
35
+ # archive_command = '/path/to/postgres_home/bin/wal_shipper.rb %p'
36
+ #
37
+ # Passwordless ssh keys need to be set up for the postgres user on all
38
+ # participating masters and slaves.
39
+ #
40
+ # You can use any replay method of your choosing on the slaves.
41
+ # Here's a nice example using pg_standby, to be put in data/recovery.conf:
42
+ #
43
+ # restore_command = 'pg_standby -t /tmp/pgrecovery.done -s5 -w0 -c /path/to/postgres_home/wal_files/ %f %p %r'
44
+ #
45
+ # Or, here's another simple alternative data/recovery.conf, for using WAL shipping
46
+ # alongside streaming replication:
47
+ #
48
+ # standby_mode = 'on'
49
+ # primary_conninfo = 'host=master.example.com port=5432 user=repl password=XXXXXXX'
50
+ # restore_command = 'cp /usr/local/pgsql/wal/%f %p'
51
+ # trigger_file = '/usr/local/pgsql/pg.become_primary'
52
+ # archive_cleanup_command = '/usr/local/bin/pg_archivecleanup /usr/local/pgsql/wal %r'
53
+ #
54
+ #========================================================================================
55
+
56
+
57
+ require 'pathname'
58
+ require 'yaml'
59
+ require 'fileutils'
60
+ require 'ostruct'
61
+
62
+
63
+ ### Encapsulate WAL shipping functionality.
64
+ ###
65
+ module WalShipper
66
+
67
+ ### Send messages to the PostgreSQL log files.
68
+ ###
69
+ def log( msg )
70
+ return unless @debug
71
+ puts "WAL Shipper: %s" % [ msg ]
72
+ end
73
+
74
+
75
+ ### An object that represents a single destination from the
76
+ ### configuration file.
77
+ ###
78
+ class Destination < OpenStruct
79
+ include WalShipper
80
+
81
+ ### Create a new WalShipper::Destination object.
82
+ def initialize( dest, debug=false )
83
+ @debug = debug
84
+ super( dest )
85
+ self.validate
86
+ end
87
+
88
+ #########
89
+ protected
90
+ #########
91
+
92
+
93
+ ### Check for required keys and normalize various keys.
94
+ ###
95
+ def validate
96
+ # Check for required destination keys
97
+ %w[ label kind ].each do |key|
98
+ if self.send( key.to_sym ).nil?
99
+ self.log "Destination %p missing required '%s' key." % [ self, key ]
100
+ self.invalid = true
101
+ end
102
+ end
103
+
104
+ # Ensure paths are Pathnames for the 'file' destination type.
105
+ self.path = Pathname.new( self.path ) if self.kind == 'file'
106
+
107
+ if self.kind == 'rsync-ssh'
108
+ self.port ||= 22
109
+ self.user = self.user ? "#{self.user}@" : ''
110
+ end
111
+ end
112
+ end # Class Destination
113
+
114
+
115
+
116
+ ### Class for creating new Destination objects and determining how to
117
+ ### ship WAL files to them.
118
+ ###
119
+ class Dispatcher
120
+ include WalShipper
121
+
122
+ ### Create a new Shipper object, given a +conf+ hash and a +wal+ file
123
+ ### Pathname object.
124
+ ###
125
+ def initialize( wal, conf )
126
+ # Make the config keys instance variables.
127
+ conf.each_pair {|key, val| self.instance_variable_set( "@#{key}", val ) }
128
+
129
+ # Spool directory check.
130
+ #
131
+ @spool = Pathname.new( @spool )
132
+ @spool.exist? or raise "The configured spool directory (%s) doesn't exist." % [ @spool ]
133
+
134
+ # Stop right away if we have disabled shipping.
135
+ #
136
+ unless @enabled
137
+ self.log "WAL shipping is disabled, queuing segment %s" % [ wal.basename ]
138
+ exit 1
139
+ end
140
+
141
+ # Instantiate Destination objects, creating new spool directories
142
+ # for each.
143
+ #
144
+ @destinations.
145
+ collect!{|dest| WalShipper::Destination.new( dest, @debug ) }.
146
+ reject {|dest| dest.invalid }.
147
+ collect do |dest|
148
+ dest.spool = @spool + dest.label
149
+ dest.spool.mkdir( 0711 ) unless dest.spool.exist?
150
+ dest
151
+ end
152
+
153
+ # Put the WAL file into the spool for processing!
154
+ #
155
+ @waldir = @spool + 'wal_segments'
156
+ @waldir.mkdir( 0711 ) unless @waldir.exist?
157
+
158
+ self.log "Copying %s to %s" % [ wal.basename, @waldir ]
159
+ FileUtils::cp wal, @waldir
160
+
161
+ # 'wal' now references the copy. The original is managed and auto-expired
162
+ # by PostgreSQL when a new checkpoint segment it reached.
163
+ @wal = @waldir + wal.basename
164
+ end
165
+
166
+
167
+ ### Create hardlinks for the WAL file into each of the destination directories
168
+ ### for separate queueing and recording of what was shipped successfully.
169
+ ###
170
+ def link
171
+ @destinations.each do |dest|
172
+ self.log "Linking %s into %s" % [ @wal.basename, dest.spool.basename ]
173
+ FileUtils::ln @wal, dest.spool, :force => true
174
+ end
175
+ end
176
+
177
+
178
+ ### Decide to be synchronous or threaded, and delegate each destination
179
+ ### to the proper ship method.
180
+ ###
181
+ def dispatch
182
+ # Synchronous mode.
183
+ #
184
+ unless @async
185
+ self.log "Performing a synchronous dispatch."
186
+ @destinations.each {|dest| self.dispatch_dest( dest ) }
187
+ return
188
+ end
189
+
190
+ tg = ThreadGroup.new
191
+
192
+ # Async, one thread per destination
193
+ #
194
+ if @async_max.nil? || @async_max.to_i.zero?
195
+ self.log "Performing an asynchronous dispatch: one thread per destination."
196
+ @destinations.each do |dest|
197
+ t = Thread.new do
198
+ Thread.current.abort_on_exception = true
199
+ self.dispatch_dest( dest )
200
+ end
201
+ tg.add( t )
202
+ end
203
+ tg.list.each {|t| t.join }
204
+ return
205
+ end
206
+
207
+ # Async, one thread per destination, in groups of asynx_max size.
208
+ #
209
+ self.log "Performing an asynchronous dispatch: one thread per destination, %d at a time." % [ @async_max ]
210
+ all_dests = @destinations.dup
211
+ dest_chunks = []
212
+ until all_dests.empty? do
213
+ dest_chunks << all_dests.slice!( 0, @async_max )
214
+ end
215
+
216
+ dest_chunks.each do |chunk|
217
+ chunk.each do |dest|
218
+ t = Thread.new do
219
+ Thread.current.abort_on_exception = true
220
+ self.dispatch_dest( dest )
221
+ end
222
+ tg.add( t )
223
+ end
224
+
225
+ tg.list.each {|t| t.join }
226
+ end
227
+
228
+ return
229
+ end
230
+
231
+
232
+ ### Remove any WAL segments no longer needed by slaves.
233
+ ###
234
+ def clean_spool
235
+ total = 0
236
+ @waldir.children.each do |wal|
237
+ if wal.stat.nlink == 1
238
+ total += wal.unlink
239
+ end
240
+ end
241
+
242
+ self.log "Removed %d WAL segment%s." % [ total, total == 1 ? '' : 's' ]
243
+ end
244
+
245
+
246
+
247
+ #########
248
+ protected
249
+ #########
250
+
251
+ ### Send WAL segments to remote +dest+ via rsync+ssh.
252
+ ### Passwordless keys between the user running this script (postmaster owner)
253
+ ### and remote user need to be set up in advance.
254
+ ###
255
+ def ship_rsync_ssh( dest )
256
+ if dest.host.nil?
257
+ self.log "Destination %p missing required 'host' key. WAL is queued." % [ dest.host ]
258
+ return
259
+ end
260
+
261
+ rsync_flags = '-zc'
262
+ ssh_string = "%s -o ConnectTimeout=%d -o StrictHostKeyChecking=no -p %d" %
263
+ [ @ssh, @ssh_timeout || 10, dest.port ]
264
+ src_string = ''
265
+ dst_string = "%s%s:%s/" % [ dest.user, dest.host, dest.path ]
266
+
267
+ # If there are numerous files in the spool dir, it means there was
268
+ # an error transferring to this host in the past. Try and ship all
269
+ # WAL segments, instead of just the new one. PostgreSQL on the slave
270
+ # side will "do the right thing" as they come in, regardless of
271
+ # ordering.
272
+ #
273
+ if dest.spool.children.length > 1
274
+ src_string = dest.spool.to_s + '/'
275
+ rsync_flags << 'r'
276
+ else
277
+ src_string = dest.spool + @wal.basename
278
+ end
279
+
280
+
281
+ ship_wal_cmd = [
282
+ @rsync,
283
+ @debug ? (rsync_flags << 'vh') : (rsync_flags << 'q'),
284
+ '--remove-source-files',
285
+ '-e', ssh_string,
286
+ src_string, dst_string
287
+ ]
288
+
289
+ self.log "Running command '%s'" % [ ship_wal_cmd.join(' ') ]
290
+ system *ship_wal_cmd
291
+
292
+ # Run external notification program on error, if one is configured.
293
+ #
294
+ unless $?.success?
295
+ self.log "Ack! Error while shipping to %p, WAL is queued." % [ dest.label ]
296
+ system @error_cmd, dest.label if @error_cmd
297
+ end
298
+ end
299
+
300
+
301
+ ### Copy WAL segments to remote path as set in +dest+.
302
+ ### This is useful for longer term PITR, copying to NFS shares, etc.
303
+ ###
304
+ def ship_file( dest )
305
+ if dest.path.nil?
306
+ self.log "Destination %p missing required 'path' key. WAL is queued." % [ dest ]
307
+ return
308
+ end
309
+ dest.path.mkdir( 0711 ) unless dest.path.exist?
310
+
311
+ # If there are numerous files in the spool dir, it means there was
312
+ # an error transferring to this host in the past. Try and ship all
313
+ # WAL segments, instead of just the new one. PostgreSQL on the slave
314
+ # side will "do the right thing" as they come in, regardless of
315
+ # ordering.
316
+ #
317
+ if dest.spool.children.length > 1
318
+ dest.spool.children.each do |wal|
319
+ wal.unlink if self.copy_file( wal, dest.path, dest.label, dest.compress )
320
+ end
321
+ else
322
+ wal = dest.spool + @wal.basename
323
+ wal.unlink if self.copy_file( wal, dest.path, dest.label, dest.compress )
324
+ end
325
+ end
326
+
327
+
328
+ ### Given a +wal+ Pathname, a +path+ destination, and the destination
329
+ ### label, copy and optionally compress a WAL file.
330
+ ###
331
+ def copy_file( wal, path, label, compress=false )
332
+ dest_file = path + wal.basename
333
+ FileUtils::cp wal, dest_file
334
+ if compress
335
+ system *[ 'gzip', '-f', dest_file ]
336
+ raise "Error while compressing: %s" % [ wal.basename ] unless $?.success?
337
+ end
338
+ self.log "Copied %s%s to %s." %
339
+ [ wal.basename, compress ? ' (and compressed)' : '', path ]
340
+ return true
341
+ rescue => err
342
+ self.log "Ack! Error while copying '%s' (%s) to %p, WAL is queued." %
343
+ [ wal.basename, err.message, path ]
344
+ system @error_cmd, label if @error_cmd
345
+ return false
346
+ end
347
+
348
+
349
+ ### Figure out how to send the WAL file to its intended destination +dest+.
350
+ ###
351
+ def dispatch_dest( dest )
352
+ if ! dest.enabled.nil? && ! dest.enabled
353
+ self.log "Skipping explicitly disabled destination %p, WAL is queued." % [ dest.label ]
354
+ return
355
+ end
356
+
357
+ # Send to the appropriate method. ( rsync-ssh --> ship_rsync_ssh )
358
+ #
359
+ meth = ( 'ship_' + dest.kind.gsub(/-/, '_') ).to_sym
360
+ if WalShipper::Dispatcher.method_defined?( meth )
361
+ self.send( meth, dest )
362
+ else
363
+ self.log "Unknown destination kind %p for %p. WAL is queued." % [ dest.kind, dest.label ]
364
+ end
365
+ end
366
+ end
367
+ end
368
+
369
+ # Ship the WAL file!
370
+ #
371
+ if __FILE__ == $0
372
+ CONFIG_DIR = Pathname.new( __FILE__ ).dirname.parent + 'etc'
373
+ CONFIG = CONFIG_DIR + 'wal_shipper.conf'
374
+
375
+ unless CONFIG.exist?
376
+ CONFIG_DIR.mkdir( 0711 ) unless CONFIG_DIR.exist?
377
+ CONFIG.open('w') {|conf| conf.print(DATA.read) }
378
+ CONFIG.chmod( 0644 )
379
+ puts "No WAL shipping configuration found, default file created."
380
+ end
381
+
382
+ wal = ARGV[0] or raise "No WAL file was specified on the command line."
383
+ wal = Pathname.new( wal )
384
+ conf = YAML.load( CONFIG.read )
385
+
386
+ shipper = WalShipper::Dispatcher.new( wal, conf )
387
+ shipper.link
388
+ shipper.dispatch
389
+ shipper.clean_spool
390
+ end
391
+
392
+
393
+ __END__
394
+ ---
395
+ # Spool from pg_xlog to the working area?
396
+ # This must be set to 'true' for wal shipping to function!
397
+ enabled: false
398
+
399
+ # Log everything to the PostgreSQL log files?
400
+ debug: true
401
+
402
+ # The working area for WAL segments.
403
+ spool: /opt/local/var/db/postgresql84/wal
404
+
405
+ # With multiple slaves, ship WAL in parallel, or be synchronous?
406
+ async: false
407
+
408
+ # Put a ceiling on the parallel threads?
409
+ # '0' or removing this option uses a thread for each destination,
410
+ # regardless of how many you have. Keep in mind that's 16 * destination
411
+ # count megs of simultaneous bandwidth.
412
+ async_max: 5
413
+
414
+ # Paths and settings for various binaries.
415
+ rsync: /usr/bin/rsync
416
+ ssh: /usr/bin/ssh
417
+ ssh_timeout: 10
418
+
419
+ destinations:
420
+
421
+ - label: rsync-example
422
+ port: 2222
423
+ kind: rsync-ssh
424
+ host: localhost
425
+ user: postgres
426
+ path: wal # relative to the user's homedir on the remote host
427
+ enabled: false
428
+
429
+ - label: file-example
430
+ kind: file
431
+ compress: true
432
+ enabled: true
433
+ path: /tmp/someplace
434
+