pg_party 1.0.1 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,6 +11,10 @@ module PgParty
11
11
  raise "#create_list_partition is not implemented"
12
12
  end
13
13
 
14
+ def create_hash_partition(*)
15
+ raise "#create_hash_partition is not implemented"
16
+ end
17
+
14
18
  def create_range_partition_of(*)
15
19
  raise "#create_range_partition_of is not implemented"
16
20
  end
@@ -19,6 +23,14 @@ module PgParty
19
23
  raise "#create_list_partition_of is not implemented"
20
24
  end
21
25
 
26
+ def create_hash_partition_of(*)
27
+ raise "#create_hash_partition_of is not implemented"
28
+ end
29
+
30
+ def create_default_partition_of(*)
31
+ raise "#create_default_partition_of is not implemented"
32
+ end
33
+
22
34
  def create_table_like(*)
23
35
  raise "#create_table_like is not implemented"
24
36
  end
@@ -31,9 +43,33 @@ module PgParty
31
43
  raise "#attach_list_partition is not implemented"
32
44
  end
33
45
 
46
+ def attach_hash_partition(*)
47
+ raise "#attach_hash_partition is not implemented"
48
+ end
49
+
50
+ def attach_default_partition(*)
51
+ raise "#attach_default_partition is not implemented"
52
+ end
53
+
34
54
  def detach_partition(*)
35
55
  raise "#detach_partition is not implemented"
36
56
  end
57
+
58
+ def parent_for_table_name(*)
59
+ raise "#parent_for_table_name is not implemented"
60
+ end
61
+
62
+ def partitions_for_table_name(*)
63
+ raise "#partitions_for_table_name is not implemented"
64
+ end
65
+
66
+ def add_index_on_all_partitions(*)
67
+ raise "#add_index_on_all_partitions is not implemented"
68
+ end
69
+
70
+ def table_partitioned?(*)
71
+ raise "#table_partitioned? is not implemented"
72
+ end
37
73
  end
38
74
  end
39
75
  end
@@ -1,41 +1,78 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "pg_party/adapter_decorator"
4
+ require "ruby2_keywords"
4
5
 
5
6
  module PgParty
6
7
  module Adapter
7
8
  module PostgreSQLMethods
8
- def create_range_partition(*args, &blk)
9
+ ruby2_keywords def create_range_partition(*args, &blk)
9
10
  PgParty::AdapterDecorator.new(self).create_range_partition(*args, &blk)
10
11
  end
11
12
 
12
- def create_list_partition(*args, &blk)
13
+ ruby2_keywords def create_list_partition(*args, &blk)
13
14
  PgParty::AdapterDecorator.new(self).create_list_partition(*args, &blk)
14
15
  end
15
16
 
16
- def create_range_partition_of(*args)
17
+ ruby2_keywords def create_hash_partition(*args, &blk)
18
+ PgParty::AdapterDecorator.new(self).create_hash_partition(*args, &blk)
19
+ end
20
+
21
+ ruby2_keywords def create_range_partition_of(*args)
17
22
  PgParty::AdapterDecorator.new(self).create_range_partition_of(*args)
18
23
  end
19
24
 
20
- def create_list_partition_of(*args)
25
+ ruby2_keywords def create_list_partition_of(*args)
21
26
  PgParty::AdapterDecorator.new(self).create_list_partition_of(*args)
22
27
  end
23
28
 
24
- def create_table_like(*args)
29
+ ruby2_keywords def create_hash_partition_of(*args)
30
+ PgParty::AdapterDecorator.new(self).create_hash_partition_of(*args)
31
+ end
32
+
33
+ ruby2_keywords def create_default_partition_of(*args)
34
+ PgParty::AdapterDecorator.new(self).create_default_partition_of(*args)
35
+ end
36
+
37
+ ruby2_keywords def create_table_like(*args)
25
38
  PgParty::AdapterDecorator.new(self).create_table_like(*args)
26
39
  end
27
40
 
28
- def attach_range_partition(*args)
41
+ ruby2_keywords def attach_range_partition(*args)
29
42
  PgParty::AdapterDecorator.new(self).attach_range_partition(*args)
30
43
  end
31
44
 
32
- def attach_list_partition(*args)
45
+ ruby2_keywords def attach_list_partition(*args)
33
46
  PgParty::AdapterDecorator.new(self).attach_list_partition(*args)
34
47
  end
35
48
 
36
- def detach_partition(*args)
49
+ ruby2_keywords def attach_hash_partition(*args)
50
+ PgParty::AdapterDecorator.new(self).attach_hash_partition(*args)
51
+ end
52
+
53
+ ruby2_keywords def attach_default_partition(*args)
54
+ PgParty::AdapterDecorator.new(self).attach_default_partition(*args)
55
+ end
56
+
57
+ ruby2_keywords def detach_partition(*args)
37
58
  PgParty::AdapterDecorator.new(self).detach_partition(*args)
38
59
  end
60
+
61
+ ruby2_keywords def partitions_for_table_name(*args)
62
+ PgParty::AdapterDecorator.new(self).partitions_for_table_name(*args)
63
+ end
64
+
65
+ ruby2_keywords def parent_for_table_name(*args)
66
+ PgParty::AdapterDecorator.new(self).parent_for_table_name(*args)
67
+ end
68
+
69
+ ruby2_keywords def add_index_on_all_partitions(*args)
70
+ PgParty::AdapterDecorator.new(self).add_index_on_all_partitions(*args)
71
+ end
72
+
73
+ ruby2_keywords def table_partitioned?(*args)
74
+ PgParty::AdapterDecorator.new(self).table_partitioned?(*args)
75
+ end
39
76
  end
40
77
  end
41
78
  end
@@ -1,10 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "digest"
4
- require "pg_party/cache"
4
+ require 'parallel'
5
5
 
6
6
  module PgParty
7
7
  class AdapterDecorator < SimpleDelegator
8
+ SUPPORTED_PARTITION_TYPES = %i[range list hash].freeze
9
+
8
10
  def initialize(adapter)
9
11
  super(adapter)
10
12
 
@@ -19,6 +21,10 @@ module PgParty
19
21
  create_partition(table_name, :list, partition_key, **options, &blk)
20
22
  end
21
23
 
24
+ def create_hash_partition(table_name, partition_key:, **options, &blk)
25
+ create_partition(table_name, :hash, partition_key, **options, &blk)
26
+ end
27
+
22
28
  def create_range_partition_of(table_name, start_range:, end_range:, **options)
23
29
  create_partition_of(table_name, range_constraint_clause(start_range, end_range), **options)
24
30
  end
@@ -27,17 +33,42 @@ module PgParty
27
33
  create_partition_of(table_name, list_constraint_clause(values), **options)
28
34
  end
29
35
 
36
+ def create_hash_partition_of(table_name, modulus:, remainder:, **options)
37
+ create_partition_of(table_name, hash_constraint_clause(modulus, remainder), **options)
38
+ end
39
+
40
+ def create_default_partition_of(table_name, **options)
41
+ create_partition_of(table_name, nil, default_partition: true, **options)
42
+ end
43
+
30
44
  def create_table_like(table_name, new_table_name, **options)
31
- primary_key = options.fetch(:primary_key) { calculate_primary_key(table_name) }
45
+ primary_key = options.fetch(:primary_key) { calculate_primary_key(table_name) }
46
+ partition_key = options.fetch(:partition_key, nil)
47
+ partition_type = options.fetch(:partition_type, nil)
48
+ create_with_pks = options.fetch(
49
+ :create_with_primary_key,
50
+ PgParty.config.create_with_primary_key
51
+ )
52
+
53
+ validate_primary_key(primary_key) unless create_with_pks
54
+ if partition_type
55
+ validate_supported_partition_type!(partition_type)
56
+ raise ArgumentError, '`partition_key` is required when specifying a partition_type' unless partition_key
57
+ end
32
58
 
33
- validate_primary_key(primary_key)
59
+ like_option = if !partition_type || create_with_pks
60
+ 'INCLUDING ALL'
61
+ else
62
+ 'INCLUDING ALL EXCLUDING INDEXES'
63
+ end
34
64
 
35
65
  execute(<<-SQL)
36
66
  CREATE TABLE #{quote_table_name(new_table_name)} (
37
- LIKE #{quote_table_name(table_name)} INCLUDING ALL
38
- )
67
+ LIKE #{quote_table_name(table_name)} #{like_option}
68
+ ) #{partition_type ? partition_by_clause(partition_type, partition_key) : nil}
39
69
  SQL
40
70
 
71
+ return if partition_type
41
72
  return if !primary_key
42
73
  return if has_primary_key?(new_table_name)
43
74
 
@@ -55,32 +86,124 @@ module PgParty
55
86
  attach_partition(parent_table_name, child_table_name, list_constraint_clause(values))
56
87
  end
57
88
 
89
+ def attach_hash_partition(parent_table_name, child_table_name, modulus:, remainder:)
90
+ attach_partition(parent_table_name, child_table_name, hash_constraint_clause(modulus, remainder))
91
+ end
92
+
93
+ def attach_default_partition(parent_table_name, child_table_name)
94
+ execute(<<-SQL)
95
+ ALTER TABLE #{quote_table_name(parent_table_name)}
96
+ ATTACH PARTITION #{quote_table_name(child_table_name)}
97
+ DEFAULT
98
+ SQL
99
+
100
+ PgParty.cache.clear!
101
+ end
102
+
58
103
  def detach_partition(parent_table_name, child_table_name)
59
104
  execute(<<-SQL)
60
105
  ALTER TABLE #{quote_table_name(parent_table_name)}
61
106
  DETACH PARTITION #{quote_table_name(child_table_name)}
62
107
  SQL
63
108
 
64
- PgParty::Cache.clear!
109
+ PgParty.cache.clear!
65
110
  end
66
111
 
67
- private
112
+ def partitions_for_table_name(table_name, include_subpartitions:, _accumulator: [])
113
+ select_values(%[
114
+ SELECT pg_inherits.inhrelid::regclass::text
115
+ FROM pg_tables
116
+ INNER JOIN pg_inherits
117
+ ON pg_tables.tablename::regclass = pg_inherits.inhparent::regclass
118
+ WHERE pg_tables.schemaname = current_schema() AND
119
+ pg_tables.tablename = #{quote(table_name)}
120
+ ]).each_with_object(_accumulator) do |partition, acc|
121
+ acc << partition
122
+ next unless include_subpartitions
123
+
124
+ partitions_for_table_name(partition, include_subpartitions: true, _accumulator: acc)
125
+ end
126
+ end
68
127
 
69
- def create_partition(table_name, type, partition_key, **options)
70
- modified_options = options.except(:id, :primary_key, :template)
71
- template = options.fetch(:template, true)
72
- id = options.fetch(:id, :bigserial)
73
- primary_key = options.fetch(:primary_key) { calculate_primary_key(table_name) }
128
+ def parent_for_table_name(table_name, traverse: false)
129
+ parent = select_values(%[
130
+ SELECT pg_inherits.inhparent::regclass::text
131
+ FROM pg_tables
132
+ INNER JOIN pg_inherits
133
+ ON pg_tables.tablename::regclass = pg_inherits.inhrelid::regclass
134
+ WHERE pg_tables.schemaname = current_schema() AND
135
+ pg_tables.tablename = #{quote(table_name)}
136
+ ]).first
137
+ return parent if parent.nil? || !traverse
138
+
139
+ while (parents_parent = parent_for_table_name(parent)) do
140
+ parent = parents_parent
141
+ end
142
+
143
+ parent
144
+ end
74
145
 
75
- validate_primary_key(primary_key)
146
+ def add_index_on_all_partitions(table_name, column_name, in_threads: nil, **options)
147
+ if in_threads && open_transactions > 0
148
+ raise ArgumentError, '`in_threads:` cannot be used within a transaction. If running in a migration, use '\
149
+ '`disable_ddl_transaction!` and break out this operation into its own migration.'
150
+ end
76
151
 
77
- modified_options[:id] = false
78
- modified_options[:options] = "PARTITION BY #{type.to_s.upcase} (#{quote_partition_key(partition_key)})"
152
+ index_name, index_type, index_columns, index_options, algorithm, using = add_index_options(
153
+ table_name, column_name, options
154
+ )
155
+
156
+ # Postgres limits index name to 63 bytes (characters). We will use 8 characters for a `_random_suffix`
157
+ # on partitions to ensure no conflicts, leaving 55 chars for the specified index name
158
+ raise ArgumentError 'index name is too long - must be 55 characters or fewer' if index_name.length > 55
159
+
160
+ recursive_add_index(
161
+ table_name: table_name,
162
+ index_name: index_name,
163
+ index_type: index_type,
164
+ index_columns: index_columns,
165
+ index_options: index_options,
166
+ algorithm: algorithm,
167
+ using: using,
168
+ in_threads: in_threads
169
+ )
170
+ end
171
+
172
+ def table_partitioned?(table_name)
173
+ select_values(%[
174
+ SELECT relkind FROM pg_catalog.pg_class AS c
175
+ JOIN pg_catalog.pg_namespace AS ns ON c.relnamespace = ns.oid
176
+ WHERE relname = #{quote(table_name)} AND nspname = current_schema()
177
+ ]).first == 'p'
178
+ end
179
+
180
+ private
181
+
182
+ def create_partition(table_name, type, partition_key, **options)
183
+ modified_options = options.except(:id, :primary_key, :template, :create_with_primary_key)
184
+ template = options.fetch(:template, PgParty.config.create_template_tables)
185
+ id = options.fetch(:id, :bigserial)
186
+ primary_key = options.fetch(:primary_key) { calculate_primary_key(table_name) }
187
+ create_with_pks = options.fetch(
188
+ :create_with_primary_key,
189
+ PgParty.config.create_with_primary_key
190
+ )
191
+
192
+ validate_supported_partition_type!(type)
193
+
194
+ if create_with_pks
195
+ modified_options[:primary_key] = primary_key
196
+ modified_options[:id] = id
197
+ else
198
+ validate_primary_key(primary_key)
199
+ modified_options[:id] = false
200
+ end
201
+ modified_options[:options] = partition_by_clause(type, partition_key)
79
202
 
80
203
  create_table(table_name, modified_options) do |td|
81
- if id == :uuid
204
+ if !modified_options[:id] && id == :uuid
82
205
  td.column(primary_key, id, null: false, default: uuid_function)
83
- elsif id
206
+ elsif !modified_options[:id] && id
84
207
  td.column(primary_key, id, null: false)
85
208
  end
86
209
 
@@ -88,11 +211,16 @@ module PgParty
88
211
  end
89
212
 
90
213
  # Rails 4 has a bug where uuid columns are always nullable
91
- change_column_null(table_name, primary_key, false) if id == :uuid
214
+ change_column_null(table_name, primary_key, false) if !modified_options[:id] && id == :uuid
92
215
 
93
216
  return unless template
94
217
 
95
- create_table_like(table_name, template_table_name(table_name), primary_key: id && primary_key)
218
+ create_table_like(
219
+ table_name,
220
+ template_table_name(table_name),
221
+ primary_key: id && primary_key,
222
+ create_with_primary_key: create_with_pks
223
+ )
96
224
  end
97
225
 
98
226
  def create_partition_of(table_name, constraint_clause, **options)
@@ -100,13 +228,21 @@ module PgParty
100
228
  primary_key = options.fetch(:primary_key) { calculate_primary_key(table_name) }
101
229
  template_table_name = template_table_name(table_name)
102
230
 
231
+ validate_default_partition_support! if options[:default_partition]
232
+
103
233
  if schema_cache.data_source_exists?(template_table_name)
104
- create_table_like(template_table_name, child_table_name, primary_key: false)
234
+ create_table_like(template_table_name, child_table_name, primary_key: false,
235
+ partition_type: options[:partition_type], partition_key: options[:partition_key])
105
236
  else
106
- create_table_like(table_name, child_table_name, primary_key: primary_key)
237
+ create_table_like(table_name, child_table_name, primary_key: primary_key,
238
+ partition_type: options[:partition_type], partition_key: options[:partition_key])
107
239
  end
108
240
 
109
- attach_partition(table_name, child_table_name, constraint_clause)
241
+ if options[:default_partition]
242
+ attach_default_partition(table_name, child_table_name)
243
+ else
244
+ attach_partition(table_name, child_table_name, constraint_clause)
245
+ end
110
246
 
111
247
  child_table_name
112
248
  end
@@ -118,7 +254,83 @@ module PgParty
118
254
  FOR VALUES #{constraint_clause}
119
255
  SQL
120
256
 
121
- PgParty::Cache.clear!
257
+ PgParty.cache.clear!
258
+ end
259
+
260
+ def recursive_add_index(table_name:, index_name:, index_type:, index_columns:, index_options:, using:, algorithm:,
261
+ in_threads: nil, _parent_index_name: nil, _created_index_names: [])
262
+ partitions = partitions_for_table_name(table_name, include_subpartitions: false)
263
+ updated_name = _created_index_names.empty? ? index_name : generate_index_name(index_name, table_name)
264
+
265
+ # If this is a partitioned table, add index ONLY on this table.
266
+ if table_partitioned?(table_name)
267
+ add_index_only(table_name, type: index_type, name: updated_name, using: using, columns: index_columns,
268
+ options: index_options)
269
+ _created_index_names << updated_name
270
+
271
+ parallel_map(partitions, in_threads: in_threads) do |partition_name|
272
+ recursive_add_index(
273
+ table_name: partition_name,
274
+ index_name: index_name,
275
+ index_type: index_type,
276
+ index_columns: index_columns,
277
+ index_options: index_options,
278
+ using: using,
279
+ algorithm: algorithm,
280
+ _parent_index_name: updated_name,
281
+ _created_index_names: _created_index_names
282
+ )
283
+ end
284
+ else
285
+ _created_index_names << updated_name # Track as created before execution of concurrent index command
286
+ add_index_from_options(table_name, name: updated_name, type: index_type, algorithm: algorithm, using: using,
287
+ columns: index_columns, options: index_options)
288
+ end
289
+
290
+ attach_child_index(updated_name, _parent_index_name) if _parent_index_name
291
+
292
+ return true if index_valid?(updated_name)
293
+
294
+ raise 'index creation failed - an index was marked invalid'
295
+ rescue => e
296
+ # Clean up any indexes created so this command can be retried later
297
+ drop_indices_if_exist(_created_index_names)
298
+ raise e
299
+ end
300
+
301
+ def attach_child_index(child, parent)
302
+ return unless postgres_major_version >= 11
303
+
304
+ execute "ALTER INDEX #{quote_column_name(parent)} ATTACH PARTITION #{quote_column_name(child)}"
305
+ end
306
+
307
+ def add_index_only(table_name, type:, name:, using:, columns:, options:)
308
+ return unless postgres_major_version >= 11
309
+
310
+ execute "CREATE #{type} INDEX #{quote_column_name(name)} ON ONLY "\
311
+ "#{quote_table_name(table_name)} #{using} (#{columns})#{options}"
312
+ end
313
+
314
+ def add_index_from_options(table_name, name:, type:, algorithm:, using:, columns:, options:)
315
+ execute "CREATE #{type} INDEX #{algorithm} #{quote_column_name(name)} ON "\
316
+ "#{quote_table_name(table_name)} #{using} (#{columns})#{options}"
317
+ end
318
+
319
+ def drop_indices_if_exist(index_names)
320
+ index_names.uniq.each { |name| execute "DROP INDEX IF EXISTS #{quote_column_name(name)}" }
321
+ end
322
+
323
+ def parallel_map(arr, in_threads:)
324
+ return [] if arr.empty?
325
+ return arr.map { |item| yield(item) } unless in_threads && in_threads > 1
326
+
327
+ if ActiveRecord::Base.connection_pool.size <= in_threads
328
+ raise ArgumentError, 'in_threads: must be lower than your database connection pool size'
329
+ end
330
+
331
+ Parallel.map(arr, in_threads: in_threads) do |item|
332
+ ActiveRecord::Base.connection_pool.with_connection { yield(item) }
333
+ end
122
334
  end
123
335
 
124
336
  # Rails 5.2 now returns boolean literals
@@ -149,7 +361,7 @@ module PgParty
149
361
  if key.is_a?(Proc)
150
362
  key.call.to_s # very difficult to determine how to sanitize a complex expression
151
363
  else
152
- quote_column_name(key)
364
+ Array.wrap(key).map(&method(:quote_column_name)).join(",")
153
365
  end
154
366
  end
155
367
 
@@ -158,27 +370,69 @@ module PgParty
158
370
  end
159
371
 
160
372
  def template_table_name(table_name)
161
- "#{table_name}_template"
373
+ "#{parent_for_table_name(table_name, traverse: true) || table_name}_template"
162
374
  end
163
375
 
164
376
  def range_constraint_clause(start_range, end_range)
165
377
  "FROM (#{quote_collection(start_range)}) TO (#{quote_collection(end_range)})"
166
378
  end
167
379
 
380
+ def hash_constraint_clause(modulus, remainder)
381
+ "WITH (MODULUS #{modulus.to_i}, REMAINDER #{remainder.to_i})"
382
+ end
383
+
168
384
  def list_constraint_clause(values)
169
385
  "IN (#{quote_collection(values.try(:to_a) || values)})"
170
386
  end
171
387
 
388
+ def partition_by_clause(type, partition_key)
389
+ "PARTITION BY #{type.to_s.upcase} (#{quote_partition_key(partition_key)})"
390
+ end
391
+
172
392
  def uuid_function
173
393
  try(:supports_pgcrypto_uuid?) ? "gen_random_uuid()" : "uuid_generate_v4()"
174
394
  end
175
395
 
176
396
  def hashed_table_name(table_name, key)
177
- "#{table_name}_#{Digest::MD5.hexdigest(key)[0..6]}"
397
+ return "#{table_name}_#{Digest::MD5.hexdigest(key)[0..6]}" if key
398
+
399
+ # use _default suffix for default partitions (without a constraint clause)
400
+ "#{table_name}_default"
401
+ end
402
+
403
+ def index_valid?(index_name)
404
+ select_values(
405
+ "SELECT relname FROM pg_class, pg_index WHERE pg_index.indisvalid = false AND "\
406
+ "pg_index.indexrelid = pg_class.oid AND relname = #{quote(index_name)}"
407
+ ).empty?
408
+ end
409
+
410
+ def generate_index_name(index_name, table_name)
411
+ "#{index_name}_#{Digest::MD5.hexdigest(table_name)[0..6]}"
412
+ end
413
+
414
+ def validate_supported_partition_type!(partition_type)
415
+ if (sym = partition_type.to_s.downcase.to_sym) && sym.in?(SUPPORTED_PARTITION_TYPES)
416
+ return if sym != :hash || postgres_major_version >= 11
417
+
418
+ raise NotImplementedError, 'Hash partitions are only available in Postgres 11 or higher'
419
+ end
420
+
421
+ raise ArgumentError, "Supported partition types are #{SUPPORTED_PARTITION_TYPES.join(', ')}"
422
+ end
423
+
424
+ def validate_default_partition_support!
425
+ return if postgres_major_version >= 11
426
+
427
+ raise NotImplementedError, 'Default partitions are only available in Postgres 11 or higher'
178
428
  end
179
429
 
180
430
  def supports_partitions?
181
- __getobj__.send(:postgresql_version) >= 100000
431
+ postgres_major_version >= 10
432
+ end
433
+
434
+ def postgres_major_version
435
+ __getobj__.send(:postgresql_version)/10000
182
436
  end
183
437
  end
184
438
  end