cipherstash-pg 1.0.0.beta.1-arm64-darwin-22 → 1.0.0.beta.4-arm64-darwin-22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/.appveyor.yml +42 -0
  3. data/.gems +6 -0
  4. data/.gemtest +0 -0
  5. data/.github/workflows/binary-gems.yml +117 -0
  6. data/.github/workflows/source-gem.yml +137 -0
  7. data/.gitignore +19 -0
  8. data/.hgsigs +34 -0
  9. data/.hgtags +41 -0
  10. data/.irbrc +23 -0
  11. data/.pryrc +23 -0
  12. data/.tm_properties +21 -0
  13. data/.travis.yml +49 -0
  14. data/Gemfile +3 -3
  15. data/Gemfile.lock +45 -0
  16. data/{History.rdoc → History.md} +168 -153
  17. data/README.ja.md +266 -0
  18. data/README.md +272 -0
  19. data/Rakefile +65 -104
  20. data/Rakefile.cross +298 -0
  21. data/certs/larskanis-2023.pem +24 -0
  22. data/cipherstash-pg.gemspec +0 -0
  23. data/lib/2.7/pg_ext.bundle +0 -0
  24. data/lib/3.0/pg_ext.bundle +0 -0
  25. data/lib/3.1/pg_ext.bundle +0 -0
  26. data/lib/3.2/pg_ext.bundle +0 -0
  27. data/lib/cipherstash-pg/basic_type_map_based_on_result.rb +11 -0
  28. data/lib/cipherstash-pg/basic_type_map_for_queries.rb +113 -0
  29. data/lib/cipherstash-pg/basic_type_map_for_results.rb +30 -0
  30. data/lib/cipherstash-pg/basic_type_registry.rb +206 -0
  31. data/lib/cipherstash-pg/binary_decoder.rb +21 -0
  32. data/lib/cipherstash-pg/coder.rb +82 -0
  33. data/lib/cipherstash-pg/connection.rb +467 -0
  34. data/lib/cipherstash-pg/constants.rb +3 -0
  35. data/lib/cipherstash-pg/exceptions.rb +19 -0
  36. data/lib/cipherstash-pg/result.rb +22 -0
  37. data/lib/cipherstash-pg/text_decoder.rb +43 -0
  38. data/lib/cipherstash-pg/text_encoder.rb +67 -0
  39. data/lib/cipherstash-pg/tuple.rb +24 -0
  40. data/lib/cipherstash-pg/type_map_by_column.rb +11 -0
  41. data/lib/cipherstash-pg/version.rb +3 -0
  42. data/lib/cipherstash-pg.rb +56 -11
  43. data/lib/libpq.5.dylib +0 -0
  44. data/misc/openssl-pg-segfault.rb +15 -25
  45. data/misc/postgres/Rakefile +13 -20
  46. data/misc/postgres/lib/postgres.rb +10 -14
  47. data/misc/ruby-pg/Rakefile +13 -20
  48. data/misc/ruby-pg/lib/ruby/pg.rb +10 -14
  49. data/rakelib/task_extension.rb +17 -31
  50. data/sample/array_insert.rb +7 -20
  51. data/sample/async_api.rb +54 -96
  52. data/sample/async_copyto.rb +20 -35
  53. data/sample/async_mixed.rb +22 -50
  54. data/sample/check_conn.rb +8 -20
  55. data/sample/copydata.rb +18 -68
  56. data/sample/copyfrom.rb +26 -78
  57. data/sample/copyto.rb +10 -16
  58. data/sample/cursor.rb +9 -19
  59. data/sample/disk_usage_report.rb +89 -174
  60. data/sample/issue-119.rb +45 -93
  61. data/sample/losample.rb +48 -66
  62. data/sample/minimal-testcase.rb +6 -17
  63. data/sample/notify_wait.rb +21 -67
  64. data/sample/pg_statistics.rb +100 -281
  65. data/sample/replication_monitor.rb +119 -218
  66. data/sample/test_binary_values.rb +14 -30
  67. data/sample/wal_shipper.rb +199 -431
  68. data/sample/warehouse_partitions.rb +157 -307
  69. data/translation/.po4a-version +7 -0
  70. data/translation/po/all.pot +875 -0
  71. data/translation/po/ja.po +868 -0
  72. data/translation/po4a.cfg +9 -0
  73. metadata +50 -28
  74. data/README.ja.rdoc +0 -13
  75. data/README.rdoc +0 -233
  76. data/lib/pg/basic_type_map_based_on_result.rb +0 -47
  77. data/lib/pg/basic_type_map_for_queries.rb +0 -193
  78. data/lib/pg/basic_type_map_for_results.rb +0 -81
  79. data/lib/pg/basic_type_registry.rb +0 -301
  80. data/lib/pg/binary_decoder.rb +0 -23
  81. data/lib/pg/coder.rb +0 -104
  82. data/lib/pg/connection.rb +0 -878
  83. data/lib/pg/constants.rb +0 -12
  84. data/lib/pg/exceptions.rb +0 -18
  85. data/lib/pg/result.rb +0 -43
  86. data/lib/pg/text_decoder.rb +0 -46
  87. data/lib/pg/text_encoder.rb +0 -59
  88. data/lib/pg/tuple.rb +0 -30
  89. data/lib/pg/type_map_by_column.rb +0 -16
  90. data/lib/pg/version.rb +0 -4
  91. data/lib/pg.rb +0 -55
@@ -1,311 +1,161 @@
1
- # -*- ruby -*-
2
- # vim: set nosta noet ts=4 sw=4:
3
- #
4
- # Script to automatically move partitioned tables and their indexes
5
- # to a separate area on disk.
6
- #
7
- # Mahlon E. Smith <mahlon@martini.nu>
8
- #
9
- # Example use case:
10
- #
11
- # - You've got a heavy insert table, such as syslog data.
12
- # - This table has a partitioning trigger (or is manually partitioned)
13
- # by date, to separate incoming stuff from archival/report stuff.
14
- # - You have a tablespace on cheap or slower disk (maybe even
15
- # ZFS compressed, or some such!)
16
- #
17
- # The only assumption this script makes is that your tables are dated, and
18
- # the tablespace they're moving into already exists.
19
- #
20
- # A full example, using the syslog idea from above, where each child
21
- # table is date partitioned by a convention of "syslog_YEAR-WEEKOFYEAR":
22
- #
23
- # syslog # <--- parent
24
- # syslog_2012_06 # <--- inherited
25
- # syslog_2012_07 # <--- inherited
26
- # syslog_2012_08 # <--- inherited
27
- # ...
28
- #
29
- # You'd run this script like so:
30
- #
31
- # ./warehouse_partitions.rb -F syslog_%Y_%U
32
- #
33
- # Assuming this was week 12 of the year, tables syslog_2012_06 through
34
- # syslog_2012_11 would start sequentially migrating into the tablespace
35
- # called 'warehouse'.
36
- #
37
-
38
-
39
- require 'date'
40
- require 'ostruct'
41
- require 'optparse'
42
- require 'pathname'
43
- require 'etc'
44
- require 'pg'
45
-
46
-
47
- ### A tablespace migration class.
48
- ###
1
+ require("date")
2
+ require("ostruct")
3
+ require("optparse")
4
+ require("pathname")
5
+ require("etc")
6
+ require("cipherstash-pg")
49
7
  class PGWarehouse
50
-
51
- def initialize( opts )
52
- @opts = opts
53
- @db = PG.connect(
54
- :dbname => opts.database,
55
- :host => opts.host,
56
- :port => opts.port,
57
- :user => opts.user,
58
- :password => opts.pass,
59
- :sslmode => 'prefer'
60
- )
61
- @db.exec "SET search_path TO %s" % [ opts.schema ] if opts.schema
62
-
63
- @relations = self.relations
64
- end
65
-
66
- attr_reader :db
67
-
68
- ######
69
- public
70
- ######
71
-
72
- ### Perform the tablespace moves.
73
- ###
74
- def migrate
75
- if @relations.empty?
76
- $stderr.puts 'No tables were found for warehousing.'
77
- return
78
- end
79
-
80
- $stderr.puts "Found %d relation%s to move." % [ relations.length, relations.length == 1 ? '' : 's' ]
81
- @relations.sort_by{|_,v| v[:name] }.each do |_, val|
82
- $stderr.print " - Moving table '%s' to '%s'... " % [
83
- val[:name], @opts.tablespace
84
- ]
85
-
86
- if @opts.dryrun
87
- $stderr.puts '(not really)'
88
-
89
- else
90
- age = self.timer do
91
- db.exec "ALTER TABLE %s SET TABLESPACE %s;" % [
92
- val[:name], @opts.tablespace
93
- ]
94
- end
95
- puts age
96
- end
97
-
98
- val[ :indexes ].each do |idx|
99
- $stderr.print " - Moving index '%s' to '%s'... " % [
100
- idx, @opts.tablespace
101
- ]
102
- if @opts.dryrun
103
- $stderr.puts '(not really)'
104
-
105
- else
106
- age = self.timer do
107
- db.exec "ALTER INDEX %s SET TABLESPACE %s;" % [
108
- idx, @opts.tablespace
109
- ]
110
- end
111
- puts age
112
- end
113
- end
114
- end
115
- end
116
-
117
-
118
- #########
119
- protected
120
- #########
121
-
122
- ### Get OIDs and current tablespaces for everything under the
123
- ### specified schema.
124
- ###
125
- def relations
126
- return @relations if @relations
127
- relations = {}
128
-
129
- query = %q{
130
- SELECT c.oid AS oid,
131
- c.relname AS name,
132
- c.relkind AS kind,
133
- t.spcname AS tspace
134
- FROM pg_class AS c
135
- LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
136
- LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
137
- WHERE c.relkind = 'r' }
138
- query << "AND n.nspname='#{@opts.schema}'" if @opts.schema
139
-
140
- # Get the relations list, along with each element's current tablespace.
141
- #
142
- self.db.exec( query ) do |res|
143
- res.each do |row|
144
- relations[ row['oid'] ] = {
145
- :name => row['name'],
146
- :tablespace => row['tspace'],
147
- :indexes => [],
148
- :parent => nil
149
- }
150
- end
151
- end
152
-
153
- # Add table inheritance information.
154
- #
155
- db.exec 'SELECT inhrelid AS oid, inhparent AS parent FROM pg_inherits' do |res|
156
- res.each do |row|
157
- relations[ row['oid'] ][ :parent ] = row['parent']
158
- end
159
- end
160
-
161
- # Remove tables that don't qualify for warehousing.
162
- #
163
- # - Tables that are not children of a parent
164
- # - Tables that are already in the warehouse tablespace
165
- # - The currently active child (it's likely being written to!)
166
- # - Any table that can't be parsed into the specified format
167
- #
168
- relations.reject! do |oid, val|
169
- begin
170
- val[:parent].nil? ||
171
- val[:tablespace] == @opts.tablespace ||
172
- val[:name] == Time.now.strftime( @opts.format ) ||
173
- ! DateTime.strptime( val[:name], @opts.format )
174
- rescue ArgumentError
175
- true
176
- end
177
- end
178
-
179
- query = %q{
180
- SELECT c.oid AS oid,
181
- i.indexname AS name
182
- FROM pg_class AS c
183
- INNER JOIN pg_indexes AS i
184
- ON i.tablename = c.relname }
185
- query << "AND i.schemaname='#{@opts.schema}'" if @opts.schema
186
-
187
- # Attach index names to tables.
188
- #
189
- db.exec( query ) do |res|
190
- res.each do |row|
191
- relations[ row['oid'] ][ :indexes ] << row['name'] if relations[ row['oid'] ]
192
- end
193
- end
194
-
195
- return relations
196
- end
197
-
198
-
199
- ### Wrap arbitrary commands in a human readable timer.
200
- ###
201
- def timer
202
- start = Time.now
203
- yield
204
- age = Time.now - start
205
-
206
- diff = age
207
- secs = diff % 60
208
- diff = ( diff - secs ) / 60
209
- mins = diff % 60
210
- diff = ( diff - mins ) / 60
211
- hour = diff % 24
212
-
213
- return "%02d:%02d:%02d" % [ hour, mins, secs ]
214
- end
8
+ def initialize(opts)
9
+ @opts = opts
10
+ @db = CipherStashPG.connect(:dbname => opts.database, :host => opts.host, :port => opts.port, :user => opts.user, :password => opts.pass, :sslmode => "prefer")
11
+ @db.exec(("SET search_path TO %s" % [opts.schema])) if opts.schema
12
+ @relations = self.relations
13
+ end
14
+
15
+ attr_reader(:db)
16
+
17
+ public
18
+
19
+ def migrate
20
+ if @relations.empty? then
21
+ $stderr.puts("No tables were found for warehousing.")
22
+ return
23
+ end
24
+ $stderr.puts(("Found %d relation%s to move." % [relations.length, (relations.length == 1) ? ("") : ("s")]))
25
+ @relations.sort_by { |_, v| v[:name] }.each do |_, val|
26
+ $stderr.print((" - Moving table '%s' to '%s'... " % [val[:name], @opts.tablespace]))
27
+ if @opts.dryrun then
28
+ $stderr.puts("(not really)")
29
+ else
30
+ age = self.timer do
31
+ db.exec(("ALTER TABLE %s SET TABLESPACE %s;" % [val[:name], @opts.tablespace]))
32
+ end
33
+ puts(age)
34
+ end
35
+ val[:indexes].each do |idx|
36
+ $stderr.print((" - Moving index '%s' to '%s'... " % [idx, @opts.tablespace]))
37
+ if @opts.dryrun then
38
+ $stderr.puts("(not really)")
39
+ else
40
+ age = self.timer do
41
+ db.exec(("ALTER INDEX %s SET TABLESPACE %s;" % [idx, @opts.tablespace]))
42
+ end
43
+ puts(age)
44
+ end
45
+ end
46
+ end
47
+ end
48
+
49
+ protected
50
+
51
+ def relations
52
+ return @relations if @relations
53
+ relations = {}
54
+ query = "\n\t\t\tSELECT c.oid AS oid,\n\t\t\t\tc.relname AS name,\n\t\t\t\tc.relkind AS kind,\n\t\t\t\tt.spcname AS tspace\n\t\t\tFROM pg_class AS c\n\t\t\tLEFT JOIN pg_namespace n ON n.oid = c.relnamespace\n\t\t\tLEFT JOIN pg_tablespace t ON t.oid = c.reltablespace\n\t\t\tWHERE c.relkind = 'r' "
55
+ (query << "AND n.nspname='#{@opts.schema}'") if @opts.schema
56
+ self.db.exec(query) do |res|
57
+ res.each do |row|
58
+ relations[row["oid"]] = { :name => row["name"], :tablespace => row["tspace"], :indexes => ([]), :parent => nil }
59
+ end
60
+ end
61
+ db.exec("SELECT inhrelid AS oid, inhparent AS parent FROM pg_inherits") do |res|
62
+ res.each { |row| relations[row["oid"]][:parent] = row["parent"] }
63
+ end
64
+ relations.reject! do |oid, val|
65
+ begin
66
+ (val[:parent].nil? or ((val[:tablespace] == @opts.tablespace) or ((val[:name] == Time.now.strftime(@opts.format)) or (not DateTime.strptime(val[:name], @opts.format)))))
67
+ rescue ArgumentError
68
+ true
69
+ end
70
+ end
71
+ query = "\n\t\t\tSELECT c.oid AS oid,\n\t\t\t\ti.indexname AS name\n\t\t\tFROM pg_class AS c\n\t\t\tINNER JOIN pg_indexes AS i\n\t\t\t\tON i.tablename = c.relname "
72
+ (query << "AND i.schemaname='#{@opts.schema}'") if @opts.schema
73
+ db.exec(query) do |res|
74
+ res.each do |row|
75
+ (relations[row["oid"]][:indexes] << row["name"]) if relations[row["oid"]]
76
+ end
77
+ end
78
+ return relations
79
+ end
80
+
81
+ def timer
82
+ start = Time.now
83
+ yield
84
+ age = (Time.now - start)
85
+ diff = age
86
+ secs = (diff % 60)
87
+ diff = ((diff - secs) / 60)
88
+ mins = (diff % 60)
89
+ diff = ((diff - mins) / 60)
90
+ hour = (diff % 24)
91
+ return ("%02d:%02d:%02d" % [hour, mins, secs])
92
+ end
215
93
  end
216
-
217
-
218
- ### Parse command line arguments. Return a struct of global options.
219
- ###
220
- def parse_args( args )
221
- options = OpenStruct.new
222
- options.database = Etc.getpwuid( Process.uid ).name
223
- options.host = '127.0.0.1'
224
- options.port = 5432
225
- options.user = Etc.getpwuid( Process.uid ).name
226
- options.sslmode = 'prefer'
227
- options.tablespace = 'warehouse'
228
-
229
- opts = OptionParser.new do |opts|
230
- opts.banner = "Usage: #{$0} [options]"
231
-
232
- opts.separator ''
233
- opts.separator 'Connection options:'
234
-
235
- opts.on( '-d', '--database DBNAME',
236
- "specify the database to connect to (default: \"#{options.database}\")" ) do |db|
237
- options.database = db
238
- end
239
-
240
- opts.on( '-h', '--host HOSTNAME', 'database server host' ) do |host|
241
- options.host = host
242
- end
243
-
244
- opts.on( '-p', '--port PORT', Integer,
245
- "database server port (default: \"#{options.port}\")" ) do |port|
246
- options.port = port
247
- end
248
-
249
- opts.on( '-n', '--schema SCHEMA', String,
250
- "operate on the named schema only (default: none)" ) do |schema|
251
- options.schema = schema
252
- end
253
-
254
- opts.on( '-T', '--tablespace SPACE', String,
255
- "move old tables to this tablespace (default: \"#{options.tablespace}\")" ) do |tb|
256
- options.tablespace = tb
257
- end
258
-
259
- opts.on( '-F', '--tableformat FORMAT', String,
260
- "The naming format (strftime) for the inherited tables (default: none)" ) do |format|
261
- options.format = format
262
- end
263
-
264
- opts.on( '-U', '--user NAME',
265
- "database user name (default: \"#{options.user}\")" ) do |user|
266
- options.user = user
267
- end
268
-
269
- opts.on( '-W', 'force password prompt' ) do |pw|
270
- print 'Password: '
271
- begin
272
- system 'stty -echo'
273
- options.pass = gets.chomp
274
- ensure
275
- system 'stty echo'
276
- puts
277
- end
278
- end
279
-
280
- opts.separator ''
281
- opts.separator 'Other options:'
282
-
283
- opts.on_tail( '--dry-run', "don't actually do anything" ) do
284
- options.dryrun = true
285
- end
286
-
287
- opts.on_tail( '--help', 'show this help, then exit' ) do
288
- $stderr.puts opts
289
- exit
290
- end
291
-
292
- opts.on_tail( '--version', 'output version information, then exit' ) do
293
- puts Stats::VERSION
294
- exit
295
- end
296
- end
297
-
298
- opts.parse!( args )
299
- return options
94
+ def parse_args(args)
95
+ options = OpenStruct.new
96
+ options.database = Etc.getpwuid(Process.uid).name
97
+ options.host = "127.0.0.1"
98
+ options.port = 5432
99
+ options.user = Etc.getpwuid(Process.uid).name
100
+ options.sslmode = "prefer"
101
+ options.tablespace = "warehouse"
102
+ opts = OptionParser.new do |opts|
103
+ opts.banner = "Usage: #{$0} [options]"
104
+ opts.separator("")
105
+ opts.separator("Connection options:")
106
+ opts.on("-d", "--database DBNAME", "specify the database to connect to (default: \"#{options.database}\")") do |db|
107
+ options.database = db
108
+ end
109
+ opts.on("-h", "--host HOSTNAME", "database server host") do |host|
110
+ options.host = host
111
+ end
112
+ opts.on("-p", "--port PORT", Integer, "database server port (default: \"#{options.port}\")") do |port|
113
+ options.port = port
114
+ end
115
+ opts.on("-n", "--schema SCHEMA", String, "operate on the named schema only (default: none)") do |schema|
116
+ options.schema = schema
117
+ end
118
+ opts.on("-T", "--tablespace SPACE", String, "move old tables to this tablespace (default: \"#{options.tablespace}\")") do |tb|
119
+ options.tablespace = tb
120
+ end
121
+ opts.on("-F", "--tableformat FORMAT", String, "The naming format (strftime) for the inherited tables (default: none)") do |format|
122
+ options.format = format
123
+ end
124
+ opts.on("-U", "--user NAME", "database user name (default: \"#{options.user}\")") do |user|
125
+ options.user = user
126
+ end
127
+ opts.on("-W", "force password prompt") do |pw|
128
+ print("Password: ")
129
+ begin
130
+ (system("stty -echo")
131
+ options.pass = gets.chomp)
132
+ ensure
133
+ (system("stty echo")
134
+ puts)
135
+ end
136
+ end
137
+ opts.separator("")
138
+ opts.separator("Other options:")
139
+ opts.on_tail("--dry-run", "don't actually do anything") do
140
+ options.dryrun = true
141
+ end
142
+ opts.on_tail("--help", "show this help, then exit") do
143
+ $stderr.puts(opts)
144
+ exit
145
+ end
146
+ opts.on_tail("--version", "output version information, then exit") do
147
+ puts(Stats::VERSION)
148
+ exit
149
+ end
150
+ end
151
+ opts.parse!(args)
152
+ return options
300
153
  end
301
-
302
-
303
- if __FILE__ == $0
304
- opts = parse_args( ARGV )
305
- raise ArgumentError, "A naming format (-F) is required." unless opts.format
306
-
307
- $stdout.sync = true
308
- PGWarehouse.new( opts ).migrate
154
+ if ("(string)" == $0) then
155
+ opts = parse_args(ARGV)
156
+ unless opts.format then
157
+ raise(ArgumentError, "A naming format (-F) is required.")
158
+ end
159
+ $stdout.sync = true
160
+ PGWarehouse.new(opts).migrate
309
161
  end
310
-
311
-
@@ -0,0 +1,7 @@
1
+ po4a version 0.69.
2
+ Written by Martin Quinson and Denis Barbier.
3
+
4
+ Copyright © 2002-2022 Software in the Public Interest, Inc.
5
+ This is free software; see source code for copying
6
+ conditions. There is NO warranty; not even for
7
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.