pg_ha_migrations 1.7.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9111677a3084d7b769a43f1c7078822c9ae84420443893f54dc1e228bfa037d1
4
- data.tar.gz: 7f1db28ace4c6416980c1f5c3461669f43bd7cb116996c2af7759d48515a8899
3
+ metadata.gz: 6858f02b9a874bbaf79865c789a490c9aa1140240537d0eded8f48486c6348f3
4
+ data.tar.gz: 733394b7f83821f71821816777765d982d1580761522e0751534d3e3c62f598b
5
5
  SHA512:
6
- metadata.gz: 30c567438be90db49faf206696a51176ea397d448913194ac7b2de02349294cced3d3c94fc08e3b5225eb7b1678d997af716de4f3da9cf15491745ae92af2a22
7
- data.tar.gz: dba6fa0e40b690a838a2ae26bd48a0cf18b0acda0922a74a14ae8dec295f1c3a48181a11f0ed4bc6cb46f24e9dadd03dcad86ec805c83c00ecd0cf998176cead
6
+ metadata.gz: 55f3f8e3730fc11183e71cbf6ba9d32dabf1c8b0ec532da6861b47d045cffb348184350831f564999ecee22b601931ab577ca20f8c4e831e0d4b776babe21a58
7
+ data.tar.gz: e260ebe93cafba7f3119c41bd2594a797293713f95d072378dbc5733f176d86ce8a93c8a24b53b6fbb9ec77756f4db9ab5fdb92f6080de6ac0fb56e158e5b5d3
@@ -10,13 +10,15 @@ jobs:
10
10
  - 13
11
11
  - 14
12
12
  - 15
13
+ - 16
13
14
  ruby:
14
- - 3.0
15
- - 3.1
16
- - 3.2
15
+ - "3.0"
16
+ - "3.1"
17
+ - "3.2"
17
18
  gemfile:
18
19
  - rails_6.1
19
20
  - rails_7.0
21
+ - rails_7.1
20
22
  name: PostgreSQL ${{ matrix.pg }} - Ruby ${{ matrix.ruby }} - ${{ matrix.gemfile }}
21
23
  runs-on: ubuntu-latest
22
24
  env: # $BUNDLE_GEMFILE must be set at the job level, so it is set for all steps
data/Appraisals CHANGED
@@ -1,7 +1,11 @@
1
1
  appraise "rails-6.1" do
2
- gem "rails", "6.1.0"
2
+ gem "rails", "6.1.7.6"
3
3
  end
4
4
 
5
5
  appraise "rails-7.0" do
6
- gem "rails", "7.0.1"
6
+ gem "rails", "7.0.8"
7
+ end
8
+
9
+ appraise "rails-7.1" do
10
+ gem "rails", "7.1.0"
7
11
  end
data/Dockerfile CHANGED
@@ -6,6 +6,6 @@ RUN apt-get update && apt-get install -y curl ca-certificates gnupg lsb-release
6
6
 
7
7
  RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null
8
8
 
9
- RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
9
+ RUN echo "deb https://apt-archive.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg-archive main" > /etc/apt/sources.list.d/pgdg.list
10
10
 
11
- RUN apt update && apt-get install -y postgresql-$PG_MAJOR-partman
11
+ RUN apt update && apt-get install -y postgresql-$PG_MAJOR-partman=4.7.4-2.pgdg110+1
data/README.md CHANGED
@@ -68,7 +68,7 @@ When `unsafe_*` migration methods support checks of this type you can bypass the
68
68
 
69
69
  Similarly we believe the `force: true` option to ActiveRecord's `create_table` method is always unsafe, and therefore we disallow it even when calling `unsafe_create_table`. This option won't be enabled by default until 2.0, but you can opt-in by setting `config.allow_force_create_table = false` [in your configuration initializer](#configuration).
70
70
 
71
- [Running multiple DDL statements inside a transaction acquires exclusive locks on all of the modified objects](https://medium.com/paypal-tech/postgresql-at-scale-database-schema-changes-without-downtime-20d3749ed680#cc22). For that reason, this gem [disables DDL transactions](./lib/pg_ha_migrations.rb:8) by default. You can change this by resetting `ActiveRecord::Migration.disable_ddl_transaction` in your application.
71
+ [Running multiple DDL statements inside a transaction acquires exclusive locks on all of the modified objects](https://medium.com/paypal-tech/postgresql-at-scale-database-schema-changes-without-downtime-20d3749ed680#cc22). For that reason, this gem [disables DDL transactions](./lib/pg_ha_migrations/hacks/disable_ddl_transaction.rb) by default. You can change this by resetting `ActiveRecord::Migration.disable_ddl_transaction` in your application.
72
72
 
73
73
  The following functionality is currently unsupported:
74
74
 
@@ -76,6 +76,11 @@ The following functionality is currently unsupported:
76
76
  - Generators
77
77
  - schema.rb
78
78
 
79
+ Compatibility notes:
80
+
81
+ - While some features may work with other versions, this gem is currently tested against PostgreSQL 11+ and Partman 4.x
82
+ - There is a [bug](https://github.com/rails/rails/pull/41490) in early versions of Rails 6.1 when using `algorithm: :concurrently`. To add / remove indexes concurrently, please upgrade to at least Rails 6.1.4.
83
+
79
84
  #### safe\_create\_table
80
85
 
81
86
  Safely creates a new table.
@@ -166,6 +171,14 @@ Unsafely make a column not nullable.
166
171
  unsafe_make_column_not_nullable :table, :column
167
172
  ```
168
173
 
174
+ #### safe\_add\_index\_on\_empty\_table
175
+
176
+ Safely add an index on a table with zero rows. This will raise an error if the table contains data.
177
+
178
+ ```ruby
179
+ safe_add_index_on_empty_table :table, :column
180
+ ```
181
+
169
182
  #### safe\_add\_concurrent\_index
170
183
 
171
184
  Add an index concurrently.
@@ -188,6 +201,41 @@ Safely remove an index. Migrations that contain this statement must also include
188
201
  safe_remove_concurrent_index :table, :name => :index_name
189
202
  ```
190
203
 
204
+ #### safe\_add\_concurrent\_partitioned\_index
205
+
206
+ Add an index to a natively partitioned table concurrently, as described in the [table partitioning docs](https://www.postgresql.org/docs/current/ddl-partitioning.html):
207
+
208
+ > To avoid long lock times, it is possible to use `CREATE INDEX ON ONLY` the partitioned table; such an index is marked invalid, and the partitions do not get the index applied automatically.
209
+ > The indexes on partitions can be created individually using `CONCURRENTLY`, and then attached to the index on the parent using `ALTER INDEX .. ATTACH PARTITION`.
210
+ > Once indexes for all partitions are attached to the parent index, the parent index is marked valid automatically.
211
+
212
+ ```ruby
213
+ # Assuming this table has partitions child1 and child2, the following indexes will be created:
214
+ # - index_partitioned_table_on_column
215
+ # - index_child1_on_column (attached to index_partitioned_table_on_column)
216
+ # - index_child2_on_column (attached to index_partitioned_table_on_column)
217
+ safe_add_concurrent_partitioned_index :partitioned_table, :column
218
+ ```
219
+
220
+ Add a composite index using the `hash` index type with custom name for the parent index when the parent table contains sub-partitions.
221
+
222
+ ```ruby
223
+ # Assuming this table has partitions child1 and child2, and child1 has sub-partitions sub1 and sub2,
224
+ # the following indexes will be created:
225
+ # - custom_name_idx
226
+ # - index_child1_on_column1_column2 (attached to custom_name_idx)
227
+ # - index_sub1_on_column1_column2 (attached to index_child1_on_column1_column2)
228
+ # - index_sub2_on_column1_column2 (attached to index_child1_on_column1_column2)
229
+ # - index_child2_on_column1_column2 (attached to custom_name_idx)
230
+ safe_add_concurrent_partitioned_index :partitioned_table, [:column1, :column2], name: "custom_name_idx", using: :hash
231
+ ```
232
+
233
+ Note:
234
+
235
+ This method runs multiple DDL statements non-transactionally.
236
+ Creating or attaching an index on a child table could fail.
237
+ In such cases an exception will be raised, and an `INVALID` index will be left on the parent table.
238
+
191
239
  #### safe\_add\_unvalidated\_check\_constraint
192
240
 
193
241
  Safely add a `CHECK` constraint. The constraint will not be immediately validated on existing rows to avoid a full table scan while holding an exclusive lock. After adding the constraint, you'll need to use `safe_validate_check_constraint` to validate existing rows.
@@ -387,7 +435,7 @@ safe_partman_reapply_privileges :table
387
435
 
388
436
  #### safely\_acquire\_lock\_for\_table
389
437
 
390
- Safely acquire a lock for a table.
438
+ Safely acquire an access exclusive lock for a table.
391
439
 
392
440
  ```ruby
393
441
  safely_acquire_lock_for_table(:table) do
@@ -395,6 +443,19 @@ safely_acquire_lock_for_table(:table) do
395
443
  end
396
444
  ```
397
445
 
446
+ Safely acquire a lock for a table in a different mode.
447
+
448
+ ```ruby
449
+ safely_acquire_lock_for_table(:table, mode: :share) do
450
+ ...
451
+ end
452
+ ```
453
+
454
+ Note:
455
+
456
+ We enforce that only one table (or a table and its partitions) can be locked at a time.
457
+ Attempting to acquire a nested lock on a different table will result in an error.
458
+
398
459
  #### adjust\_lock\_timeout
399
460
 
400
461
  Adjust lock timeout.
@@ -423,6 +484,22 @@ Set maintenance work mem.
423
484
  safe_set_maintenance_work_mem_gb 1
424
485
  ```
425
486
 
487
+ #### ensure\_small\_table!
488
+
489
+ Ensure a table on disk is below the default threshold (10 megabytes).
490
+ This will raise an error if the table is too large.
491
+
492
+ ```ruby
493
+ ensure_small_table! :table
494
+ ```
495
+
496
+ Ensure a table on disk is below a custom threshold and is empty.
497
+ This will raise an error if the table is too large and/or contains data.
498
+
499
+ ```ruby
500
+ ensure_small_table! :table, empty: true, threshold: 100.megabytes
501
+ ```
502
+
426
503
  ### Configuration
427
504
 
428
505
  The gem can be configured in an initializer.
data/docker-compose.yml CHANGED
@@ -4,7 +4,7 @@ services:
4
4
  build:
5
5
  context: .
6
6
  args:
7
- - PGVERSION=${PGVERSION:-15}
7
+ - PGVERSION=${PGVERSION:-16}
8
8
  ports:
9
9
  - "5432:5432"
10
10
  environment:
@@ -2,6 +2,6 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "rails", "6.1.0"
5
+ gem "rails", "6.1.7.6"
6
6
 
7
7
  gemspec path: "../"
@@ -2,6 +2,6 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "rails", "7.0.1"
5
+ gem "rails", "7.0.8"
6
6
 
7
7
  gemspec path: "../"
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "rails", "7.1.0"
6
+
7
+ gemspec path: "../"
@@ -1,7 +1,7 @@
1
1
  require "active_record/migration/compatibility"
2
2
 
3
3
  module PgHaMigrations::AllowedVersions
4
- ALLOWED_VERSIONS = [4.2, 5.0, 5.1, 5.2, 6.0, 6.1, 7.0].map do |v|
4
+ ALLOWED_VERSIONS = [4.2, 5.0, 5.1, 5.2, 6.0, 6.1, 7.0, 7.1].map do |v|
5
5
  begin
6
6
  ActiveRecord::Migration[v]
7
7
  rescue ArgumentError
@@ -1,13 +1,18 @@
1
1
  module PgHaMigrations
2
2
  class BlockingDatabaseTransactions
3
3
  LongRunningTransaction = Struct.new(:database, :current_query, :state, :transaction_age, :tables_with_locks) do
4
+ def initialize(*args)
5
+ super
6
+
7
+ self.tables_with_locks = tables_with_locks.map { |args| Table.new(*args) }.select(&:present?)
8
+ end
9
+
4
10
  def description
5
- locked_tables = tables_with_locks.compact
6
11
  [
7
12
  database,
8
- locked_tables.size > 0 ? "tables (#{locked_tables.join(', ')})" : nil,
13
+ tables_with_locks.size > 0 ? "tables (#{tables_with_locks.map(&:fully_qualified_name).join(', ')})" : nil,
9
14
  "#{idle? ? "currently idle " : ""}transaction open for #{transaction_age}",
10
- "#{idle? ? "last " : ""}query: #{current_query}"
15
+ "#{idle? ? "last " : ""}query: #{current_query}",
11
16
  ].compact.join(" | ")
12
17
  end
13
18
 
@@ -43,7 +48,7 @@ module PgHaMigrations
43
48
  psa.#{query_column} as current_query,
44
49
  psa.state,
45
50
  clock_timestamp() - psa.xact_start AS transaction_age,
46
- array_agg(distinct c.relname) AS tables_with_locks
51
+ array_agg(distinct array[c.relname, ns.nspname, l.mode]) AS tables_with_locks
47
52
  FROM pg_stat_activity psa -- Cluster wide
48
53
  LEFT JOIN pg_locks l ON (psa.#{pid_column} = l.pid) -- Cluster wide
49
54
  LEFT JOIN pg_class c ON ( -- Database wide
@@ -56,7 +61,7 @@ module PgHaMigrations
56
61
  l.locktype != 'relation'
57
62
  OR (
58
63
  ns.nspname != 'pg_catalog'
59
- AND c.relkind = 'r'
64
+ AND c.relkind IN ('r', 'p') -- 'r' is a standard table; 'p' is a partition parent
60
65
  )
61
66
  )
62
67
  AND psa.xact_start < clock_timestamp() - ?::interval
@@ -0,0 +1,30 @@
1
+ require "active_record/connection_adapters/postgresql_adapter"
2
+ require "active_record/connection_adapters/postgresql/schema_creation"
3
+
4
+ module PgHaMigrations
5
+ module ActiveRecordHacks
6
+ module IndexAlgorithms
7
+ def index_algorithms
8
+ super.merge(only: "ONLY")
9
+ end
10
+ end
11
+
12
+ module CreateIndexDefinition
13
+ def visit_CreateIndexDefinition(o)
14
+ if o.algorithm == "ONLY"
15
+ o.algorithm = nil
16
+
17
+ quoted_index = quote_column_name(o.index.name)
18
+ quoted_table = quote_table_name(o.index.table)
19
+
20
+ super.sub("#{quoted_index} ON #{quoted_table}", "#{quoted_index} ON ONLY #{quoted_table}")
21
+ else
22
+ super
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
28
+
29
+ ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(PgHaMigrations::ActiveRecordHacks::IndexAlgorithms)
30
+ ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaCreation.prepend(PgHaMigrations::ActiveRecordHacks::CreateIndexDefinition)
@@ -12,4 +12,3 @@ module PgHaMigrations
12
12
  end
13
13
 
14
14
  ActiveRecord::Migration.singleton_class.prepend(PgHaMigrations::ActiveRecordHacks::DisableDdlTransaction)
15
-
@@ -0,0 +1,100 @@
1
+ module PgHaMigrations
2
+ class LockMode
3
+ include Comparable
4
+
5
+ MODE_CONFLICTS = ActiveSupport::OrderedHash.new
6
+
7
+ MODE_CONFLICTS[:access_share] = %i[
8
+ access_exclusive
9
+ ]
10
+
11
+ MODE_CONFLICTS[:row_share] = %i[
12
+ exclusive
13
+ access_exclusive
14
+ ]
15
+
16
+ MODE_CONFLICTS[:row_exclusive] = %i[
17
+ share
18
+ share_row_exclusive
19
+ exclusive
20
+ access_exclusive
21
+ ]
22
+
23
+ MODE_CONFLICTS[:share_update_exclusive] = %i[
24
+ share_update_exclusive
25
+ share
26
+ share_row_exclusive
27
+ exclusive
28
+ access_exclusive
29
+ ]
30
+
31
+ MODE_CONFLICTS[:share] = %i[
32
+ row_exclusive
33
+ share_update_exclusive
34
+ share_row_exclusive
35
+ exclusive
36
+ access_exclusive
37
+ ]
38
+
39
+ MODE_CONFLICTS[:share_row_exclusive] = %i[
40
+ row_exclusive
41
+ share_update_exclusive
42
+ share
43
+ share_row_exclusive
44
+ exclusive
45
+ access_exclusive
46
+ ]
47
+
48
+ MODE_CONFLICTS[:exclusive] = %i[
49
+ row_share
50
+ row_exclusive
51
+ share_update_exclusive
52
+ share
53
+ share_row_exclusive
54
+ exclusive
55
+ access_exclusive
56
+ ]
57
+
58
+ MODE_CONFLICTS[:access_exclusive] = %i[
59
+ access_share
60
+ row_share
61
+ row_exclusive
62
+ share_update_exclusive
63
+ share
64
+ share_row_exclusive
65
+ exclusive
66
+ access_exclusive
67
+ ]
68
+
69
+ attr_reader :mode
70
+
71
+ delegate :to_s, to: :mode
72
+
73
+ def initialize(mode)
74
+ @mode = mode
75
+ .to_s
76
+ .underscore
77
+ .delete_suffix("_lock")
78
+ .to_sym
79
+
80
+ if !MODE_CONFLICTS.keys.include?(@mode)
81
+ raise ArgumentError, "Unrecognized lock mode #{@mode.inspect}. Valid modes: #{MODE_CONFLICTS.keys}"
82
+ end
83
+ end
84
+
85
+ def to_sql
86
+ mode
87
+ .to_s
88
+ .upcase
89
+ .gsub("_", " ")
90
+ end
91
+
92
+ def <=>(other)
93
+ MODE_CONFLICTS.keys.index(mode) <=> MODE_CONFLICTS.keys.index(other.mode)
94
+ end
95
+
96
+ def conflicts_with?(other)
97
+ MODE_CONFLICTS[mode].include?(other.mode)
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,155 @@
1
+ module PgHaMigrations
2
+ Relation = Struct.new(:name, :schema, :mode) do
3
+ def self.connection
4
+ ActiveRecord::Base.connection
5
+ end
6
+
7
+ delegate :inspect, to: :name
8
+ delegate :connection, to: :class
9
+
10
+ def initialize(name, schema, mode=nil)
11
+ super(name, schema)
12
+
13
+ self.mode = LockMode.new(mode) if mode.present?
14
+ end
15
+
16
+ def conflicts_with?(other)
17
+ self == other && (
18
+ mode.nil? || other.mode.nil? || mode.conflicts_with?(other.mode)
19
+ )
20
+ end
21
+
22
+ def fully_qualified_name
23
+ @fully_qualified_name ||= [
24
+ PG::Connection.quote_ident(schema),
25
+ PG::Connection.quote_ident(name),
26
+ ].join(".")
27
+ end
28
+
29
+ def present?
30
+ name.present? && schema.present?
31
+ end
32
+
33
+ def ==(other)
34
+ other.is_a?(Relation) && name == other.name && schema == other.schema
35
+ end
36
+ end
37
+
38
+ class Table < Relation
39
+ def self.from_table_name(table, mode=nil)
40
+ pg_name = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(table.to_s)
41
+
42
+ schema_conditional = if pg_name.schema
43
+ "#{connection.quote(pg_name.schema)}"
44
+ else
45
+ "ANY (current_schemas(false))"
46
+ end
47
+
48
+ schema = connection.select_value(<<~SQL)
49
+ SELECT schemaname
50
+ FROM pg_tables
51
+ WHERE tablename = #{connection.quote(pg_name.identifier)} AND schemaname = #{schema_conditional}
52
+ ORDER BY array_position(current_schemas(false), schemaname)
53
+ LIMIT 1
54
+ SQL
55
+
56
+ raise UndefinedTableError, "Table #{pg_name.quoted} does not exist#{" in search path" unless pg_name.schema}" unless schema.present?
57
+
58
+ new(pg_name.identifier, schema, mode)
59
+ end
60
+
61
+ def natively_partitioned?
62
+ return @natively_partitioned if defined?(@natively_partitioned)
63
+
64
+ @natively_partitioned = !!connection.select_value(<<~SQL)
65
+ SELECT true
66
+ FROM pg_partitioned_table, pg_class, pg_namespace
67
+ WHERE pg_class.oid = pg_partitioned_table.partrelid
68
+ AND pg_class.relnamespace = pg_namespace.oid
69
+ AND pg_class.relname = #{connection.quote(name)}
70
+ AND pg_namespace.nspname = #{connection.quote(schema)}
71
+ SQL
72
+ end
73
+
74
+ def partitions(include_sub_partitions: false, include_self: false)
75
+ tables = connection.structs_from_sql(self.class, <<~SQL)
76
+ SELECT child.relname AS name, child_ns.nspname AS schema, NULLIF('#{mode}', '') AS mode
77
+ FROM pg_inherits
78
+ JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
79
+ JOIN pg_class child ON pg_inherits.inhrelid = child.oid
80
+ JOIN pg_namespace parent_ns ON parent.relnamespace = parent_ns.oid
81
+ JOIN pg_namespace child_ns ON child.relnamespace = child_ns.oid
82
+ WHERE parent.relname = #{connection.quote(name)}
83
+ AND parent_ns.nspname = #{connection.quote(schema)}
84
+ ORDER BY child.oid -- Ensure consistent ordering for tests
85
+ SQL
86
+
87
+ if include_sub_partitions
88
+ sub_partitions = tables.each_with_object([]) do |table, arr|
89
+ arr.concat(table.partitions(include_sub_partitions: true))
90
+ end
91
+
92
+ tables.concat(sub_partitions)
93
+ end
94
+
95
+ tables.prepend(self) if include_self
96
+
97
+ tables
98
+ end
99
+
100
+ def has_rows?
101
+ connection.select_value("SELECT EXISTS (SELECT 1 FROM #{fully_qualified_name} LIMIT 1)")
102
+ end
103
+
104
+ def total_bytes
105
+ connection.select_value(<<~SQL)
106
+ SELECT pg_total_relation_size(pg_class.oid)
107
+ FROM pg_class, pg_namespace
108
+ WHERE pg_class.relname = #{connection.quote(name)}
109
+ AND pg_namespace.nspname = #{connection.quote(schema)}
110
+ SQL
111
+ end
112
+ end
113
+
114
+ class Index < Relation
115
+ MAX_NAME_SIZE = 63 # bytes
116
+
117
+ def self.from_table_and_columns(table, columns)
118
+ name = connection.index_name(table.name, columns)
119
+
120
+ # modified from https://github.com/rails/rails/pull/47753
121
+ if name.bytesize > MAX_NAME_SIZE
122
+ hashed_identifier = "_#{OpenSSL::Digest::SHA256.hexdigest(name).first(10)}"
123
+ description = name.sub("index_#{table.name}_on", "idx_on")
124
+
125
+ short_limit = MAX_NAME_SIZE - hashed_identifier.bytesize
126
+ short_description = description.mb_chars.limit(short_limit).to_s
127
+
128
+ name = "#{short_description}#{hashed_identifier}"
129
+ end
130
+
131
+ new(name, table)
132
+ end
133
+
134
+ attr_accessor :table
135
+
136
+ def initialize(name, table)
137
+ super(name, table.schema)
138
+
139
+ self.table = table
140
+
141
+ connection.send(:validate_index_length!, table.name, name)
142
+ end
143
+
144
+ def valid?
145
+ !!connection.select_value(<<~SQL)
146
+ SELECT pg_index.indisvalid
147
+ FROM pg_index, pg_class, pg_namespace
148
+ WHERE pg_class.oid = pg_index.indexrelid
149
+ AND pg_class.relnamespace = pg_namespace.oid
150
+ AND pg_namespace.nspname = #{connection.quote(schema)}
151
+ AND pg_class.relname = #{connection.quote(name)}
152
+ SQL
153
+ end
154
+ end
155
+ end
@@ -1,14 +1,4 @@
1
1
  module PgHaMigrations::SafeStatements
2
- PARTITION_TYPES = %i[range list hash]
3
-
4
- PARTMAN_UPDATE_CONFIG_OPTIONS = %i[
5
- infinite_time_partitions
6
- inherit_privileges
7
- premake
8
- retention
9
- retention_keep_table
10
- ]
11
-
12
2
  def safe_added_columns_without_default_value
13
3
  @safe_added_columns_without_default_value ||= []
14
4
  end
@@ -154,6 +144,22 @@ module PgHaMigrations::SafeStatements
154
144
  end
155
145
  end
156
146
 
147
+ def safe_add_index_on_empty_table(table, columns, options={})
148
+ if options[:algorithm] == :concurrently
149
+ raise ArgumentError, "Cannot call safe_add_index_on_empty_table with :algorithm => :concurrently"
150
+ end
151
+
152
+ # Avoids taking out an unnecessary SHARE lock if the table does have data
153
+ ensure_small_table!(table, empty: true)
154
+
155
+ safely_acquire_lock_for_table(table, mode: :share) do
156
+ # Ensure data wasn't written in the split second after the first check
157
+ ensure_small_table!(table, empty: true)
158
+
159
+ unsafe_add_index(table, columns, **options)
160
+ end
161
+ end
162
+
157
163
  def safe_add_concurrent_index(table, columns, options={})
158
164
  unsafe_add_index(table, columns, **options.merge(:algorithm => :concurrently))
159
165
  end
@@ -170,6 +176,90 @@ module PgHaMigrations::SafeStatements
170
176
  unsafe_remove_index(table, **options.merge(:algorithm => :concurrently))
171
177
  end
172
178
 
179
+ def safe_add_concurrent_partitioned_index(
180
+ table,
181
+ columns,
182
+ name: nil,
183
+ if_not_exists: nil,
184
+ using: nil,
185
+ unique: nil,
186
+ where: nil,
187
+ comment: nil
188
+ )
189
+
190
+ if ActiveRecord::Base.connection.postgresql_version < 11_00_00
191
+ raise PgHaMigrations::InvalidMigrationError, "Concurrent partitioned index creation not supported on Postgres databases before version 11"
192
+ end
193
+
194
+ parent_table = PgHaMigrations::Table.from_table_name(table)
195
+
196
+ raise PgHaMigrations::InvalidMigrationError, "Table #{parent_table.inspect} is not a partitioned table" unless parent_table.natively_partitioned?
197
+
198
+ parent_index = if name.present?
199
+ PgHaMigrations::Index.new(name, parent_table)
200
+ else
201
+ PgHaMigrations::Index.from_table_and_columns(parent_table, columns)
202
+ end
203
+
204
+ # Short-circuit when if_not_exists: true and index already valid
205
+ return if if_not_exists && parent_index.valid?
206
+
207
+ child_indexes = parent_table.partitions.map do |child_table|
208
+ PgHaMigrations::Index.from_table_and_columns(child_table, columns)
209
+ end
210
+
211
+ # TODO: take out ShareLock after issue #39 is implemented
212
+ safely_acquire_lock_for_table(parent_table.fully_qualified_name) do
213
+ # CREATE INDEX ON ONLY parent_table
214
+ unsafe_add_index(
215
+ parent_table.fully_qualified_name,
216
+ columns,
217
+ name: parent_index.name,
218
+ if_not_exists: if_not_exists,
219
+ using: using,
220
+ unique: unique,
221
+ where: where,
222
+ comment: comment,
223
+ algorithm: :only, # see lib/pg_ha_migrations/hacks/add_index_on_only.rb
224
+ )
225
+ end
226
+
227
+ child_indexes.each do |child_index|
228
+ add_index_method = if child_index.table.natively_partitioned?
229
+ :safe_add_concurrent_partitioned_index
230
+ else
231
+ :safe_add_concurrent_index
232
+ end
233
+
234
+ send(
235
+ add_index_method,
236
+ child_index.table.fully_qualified_name,
237
+ columns,
238
+ name: child_index.name,
239
+ if_not_exists: if_not_exists,
240
+ using: using,
241
+ unique: unique,
242
+ where: where,
243
+ )
244
+ end
245
+
246
+ # Avoid taking out an unnecessary lock if there are no child tables to attach
247
+ if child_indexes.present?
248
+ safely_acquire_lock_for_table(parent_table.fully_qualified_name) do
249
+ child_indexes.each do |child_index|
250
+ say_with_time "Attaching index #{child_index.inspect} to #{parent_index.inspect}" do
251
+ connection.execute(<<~SQL)
252
+ ALTER INDEX #{parent_index.fully_qualified_name}
253
+ ATTACH PARTITION #{child_index.fully_qualified_name}
254
+ SQL
255
+ end
256
+ end
257
+ end
258
+ end
259
+
260
+ raise PgHaMigrations::InvalidMigrationError, "Unexpected state. Parent index #{parent_index.inspect} is invalid" unless parent_index.valid?
261
+ end
262
+
173
263
  def safe_set_maintenance_work_mem_gb(gigabytes)
174
264
  unsafe_execute("SET maintenance_work_mem = '#{PG::Connection.escape_string(gigabytes.to_s)} GB'")
175
265
  end
@@ -223,8 +313,8 @@ module PgHaMigrations::SafeStatements
223
313
  def safe_create_partitioned_table(table, partition_key:, type:, infer_primary_key: nil, **options, &block)
224
314
  raise ArgumentError, "Expected <partition_key> to be present" unless partition_key.present?
225
315
 
226
- unless PARTITION_TYPES.include?(type)
227
- raise ArgumentError, "Expected <type> to be symbol in #{PARTITION_TYPES} but received #{type.inspect}"
316
+ unless PgHaMigrations::PARTITION_TYPES.include?(type)
317
+ raise ArgumentError, "Expected <type> to be symbol in #{PgHaMigrations::PARTITION_TYPES} but received #{type.inspect}"
228
318
  end
229
319
 
230
320
  if ActiveRecord::Base.connection.postgresql_version < 10_00_00
@@ -351,7 +441,7 @@ module PgHaMigrations::SafeStatements
351
441
  retention_keep_table: retention_keep_table,
352
442
  }.compact
353
443
 
354
- unsafe_partman_update_config(create_parent_options[:parent_table], **update_config_options)
444
+ unsafe_partman_update_config(table, **update_config_options)
355
445
  end
356
446
 
357
447
  def safe_partman_update_config(table, **options)
@@ -363,7 +453,7 @@ module PgHaMigrations::SafeStatements
363
453
  end
364
454
 
365
455
  def unsafe_partman_update_config(table, **options)
366
- invalid_options = options.keys - PARTMAN_UPDATE_CONFIG_OPTIONS
456
+ invalid_options = options.keys - PgHaMigrations::PARTMAN_UPDATE_CONFIG_OPTIONS
367
457
 
368
458
  raise ArgumentError, "Unrecognized argument(s): #{invalid_options}" unless invalid_options.empty?
369
459
 
@@ -402,38 +492,13 @@ module PgHaMigrations::SafeStatements
402
492
  end
403
493
 
404
494
  def _fully_qualified_table_name_for_partman(table)
405
- identifiers = table.to_s.split(".")
406
-
407
- raise PgHaMigrations::InvalidMigrationError, "Expected table to be in the format <table> or <schema>.<table> but received #{table}" if identifiers.size > 2
408
-
409
- identifiers.each { |identifier| _validate_partman_identifier(identifier) }
410
-
411
- schema_conditional = if identifiers.size > 1
412
- "'#{identifiers.first}'"
413
- else
414
- "ANY (current_schemas(false))"
415
- end
495
+ table = PgHaMigrations::Table.from_table_name(table)
416
496
 
417
- schema = connection.select_value(<<~SQL)
418
- SELECT schemaname
419
- FROM pg_tables
420
- WHERE tablename = '#{identifiers.last}' AND schemaname = #{schema_conditional}
421
- ORDER BY array_position(current_schemas(false), schemaname)
422
- LIMIT 1
423
- SQL
424
-
425
- raise PgHaMigrations::InvalidMigrationError, "Could not find table #{table}" unless schema.present?
426
-
427
- _validate_partman_identifier(schema)
428
-
429
- # Quoting is unneeded since _validate_partman_identifier ensures the schema / table use standard naming conventions
430
- "#{schema}.#{identifiers.last}"
431
- end
432
-
433
- def _validate_partman_identifier(identifier)
434
- if identifier.to_s !~ /^[a-z_][a-z_\d]*$/
435
- raise PgHaMigrations::InvalidMigrationError, "Partman requires schema / table names to be lowercase with underscores"
436
- end
497
+ [table.schema, table.name].each do |identifier|
498
+ if identifier.to_s !~ /^[a-z_][a-z_\d]*$/
499
+ raise PgHaMigrations::InvalidMigrationError, "Partman requires schema / table names to be lowercase with underscores"
500
+ end
501
+ end.join(".")
437
502
  end
438
503
 
439
504
  def _per_migration_caller
@@ -463,17 +528,39 @@ module PgHaMigrations::SafeStatements
463
528
  super(conn, direction)
464
529
  end
465
530
 
466
- def safely_acquire_lock_for_table(table, &block)
531
+ def safely_acquire_lock_for_table(table, mode: :access_exclusive, &block)
532
+ nested_target_table = Thread.current[__method__]
533
+
467
534
  _check_postgres_adapter!
468
- table = table.to_s
469
- quoted_table_name = connection.quote_table_name(table)
535
+
536
+ target_table = PgHaMigrations::Table.from_table_name(table, mode)
537
+
538
+ if nested_target_table
539
+ if nested_target_table != target_table
540
+ raise PgHaMigrations::InvalidMigrationError, "Nested lock detected! Cannot acquire lock on #{target_table.fully_qualified_name} while #{nested_target_table.fully_qualified_name} is locked."
541
+ elsif nested_target_table.mode < target_table.mode
542
+ raise PgHaMigrations::InvalidMigrationError, "Lock escalation detected! Cannot change lock level from :#{nested_target_table.mode} to :#{target_table.mode} for #{target_table.fully_qualified_name}."
543
+ end
544
+ else
545
+ Thread.current[__method__] = target_table
546
+ end
547
+
548
+ # Locking a partitioned table will also lock child tables (including sub-partitions),
549
+ # so we need to check for blocking queries on those tables as well
550
+ target_tables = target_table.partitions(include_sub_partitions: true, include_self: true)
470
551
 
471
552
  successfully_acquired_lock = false
472
553
 
473
554
  until successfully_acquired_lock
474
555
  while (
475
556
  blocking_transactions = PgHaMigrations::BlockingDatabaseTransactions.find_blocking_transactions("#{PgHaMigrations::LOCK_TIMEOUT_SECONDS} seconds")
476
- blocking_transactions.any? { |query| query.tables_with_locks.include?(table) }
557
+ blocking_transactions.any? do |query|
558
+ query.tables_with_locks.any? do |locked_table|
559
+ target_tables.any? do |target_table|
560
+ target_table.conflicts_with?(locked_table)
561
+ end
562
+ end
563
+ end
477
564
  )
478
565
  say "Waiting on blocking transactions:"
479
566
  blocking_transactions.each do |blocking_transaction|
@@ -486,13 +573,13 @@ module PgHaMigrations::SafeStatements
486
573
  adjust_timeout_method = connection.postgresql_version >= 9_03_00 ? :adjust_lock_timeout : :adjust_statement_timeout
487
574
  begin
488
575
  method(adjust_timeout_method).call(PgHaMigrations::LOCK_TIMEOUT_SECONDS) do
489
- connection.execute("LOCK #{quoted_table_name};")
576
+ connection.execute("LOCK #{target_table.fully_qualified_name} IN #{target_table.mode.to_sql} MODE;")
490
577
  end
491
578
  successfully_acquired_lock = true
492
579
  rescue ActiveRecord::StatementInvalid => e
493
580
  if e.message =~ /PG::LockNotAvailable.+ lock timeout/ || e.message =~ /PG::QueryCanceled.+ statement timeout/
494
581
  sleep_seconds = PgHaMigrations::LOCK_FAILURE_RETRY_DELAY_MULTLIPLIER * PgHaMigrations::LOCK_TIMEOUT_SECONDS
495
- say "Timed out trying to acquire an exclusive lock on the #{quoted_table_name} table."
582
+ say "Timed out trying to acquire #{target_table.mode.to_sql} lock on the #{target_table.fully_qualified_name} table."
496
583
  say "Sleeping for #{sleep_seconds}s to allow potentially queued up queries to finish before continuing."
497
584
  sleep(sleep_seconds)
498
585
 
@@ -507,6 +594,8 @@ module PgHaMigrations::SafeStatements
507
594
  end
508
595
  end
509
596
  end
597
+ ensure
598
+ Thread.current[__method__] = nil unless nested_target_table
510
599
  end
511
600
 
512
601
  def adjust_lock_timeout(timeout_seconds = PgHaMigrations::LOCK_TIMEOUT_SECONDS, &block)
@@ -548,4 +637,16 @@ module PgHaMigrations::SafeStatements
548
637
  end
549
638
  end
550
639
  end
640
+
641
+ def ensure_small_table!(table, empty: false, threshold: PgHaMigrations::SMALL_TABLE_THRESHOLD_BYTES)
642
+ table = PgHaMigrations::Table.from_table_name(table)
643
+
644
+ if empty && table.has_rows?
645
+ raise PgHaMigrations::InvalidMigrationError, "Table #{table.inspect} has rows"
646
+ end
647
+
648
+ if table.total_bytes > threshold
649
+ raise PgHaMigrations::InvalidMigrationError, "Table #{table.inspect} is larger than #{threshold} bytes"
650
+ end
651
+ end
551
652
  end
@@ -80,6 +80,16 @@ module PgHaMigrations::UnsafeStatements
80
80
  raise PgHaMigrations::InvalidMigrationError, "ActiveRecord drops the :opclass option when supplying a string containing an expression or list of columns; instead either supply an array of columns or include the opclass in the string for each column"
81
81
  end
82
82
 
83
+ validated_table = PgHaMigrations::Table.from_table_name(table)
84
+
85
+ validated_index = if options[:name]
86
+ PgHaMigrations::Index.new(options[:name], validated_table)
87
+ else
88
+ PgHaMigrations::Index.from_table_and_columns(validated_table, column_names)
89
+ end
90
+
91
+ options[:name] = validated_index.name
92
+
83
93
  execute_ancestor_statement(:add_index, table, column_names, **options)
84
94
  end
85
95
 
@@ -1,3 +1,3 @@
1
1
  module PgHaMigrations
2
- VERSION = "1.7.0"
2
+ VERSION = "1.8.0"
3
3
  end
@@ -2,6 +2,8 @@ require "pg_ha_migrations/version"
2
2
  require "rails"
3
3
  require "active_record"
4
4
  require "active_record/migration"
5
+ require "active_record/connection_adapters/postgresql/utils"
6
+ require "active_support/core_ext/numeric/bytes"
5
7
  require "relation_to_struct"
6
8
  require "ruby2_keywords"
7
9
 
@@ -30,6 +32,17 @@ module PgHaMigrations
30
32
 
31
33
  LOCK_TIMEOUT_SECONDS = 5
32
34
  LOCK_FAILURE_RETRY_DELAY_MULTLIPLIER = 5
35
+ SMALL_TABLE_THRESHOLD_BYTES = 10.megabytes
36
+
37
+ PARTITION_TYPES = %i[range list hash]
38
+
39
+ PARTMAN_UPDATE_CONFIG_OPTIONS = %i[
40
+ infinite_time_partitions
41
+ inherit_privileges
42
+ premake
43
+ retention
44
+ retention_keep_table
45
+ ]
33
46
 
34
47
  # Safe versus unsafe in this context specifically means the following:
35
48
  # - Safe operations will not block for long periods of time.
@@ -52,11 +65,17 @@ module PgHaMigrations
52
65
 
53
66
  # This gem only supports the PostgreSQL adapter at this time.
54
67
  UnsupportedAdapter = Class.new(StandardError)
68
+
69
+ # Some methods need to inspect the attributes of a table. In such cases,
70
+ # this error will be raised if the table does not exist
71
+ UndefinedTableError = Class.new(StandardError)
55
72
  end
56
73
 
74
+ require "pg_ha_migrations/relation"
57
75
  require "pg_ha_migrations/blocking_database_transactions"
58
76
  require "pg_ha_migrations/blocking_database_transactions_reporter"
59
77
  require "pg_ha_migrations/partman_config"
78
+ require "pg_ha_migrations/lock_mode"
60
79
  require "pg_ha_migrations/unsafe_statements"
61
80
  require "pg_ha_migrations/safe_statements"
62
81
  require "pg_ha_migrations/dependent_objects_checks"
@@ -64,6 +83,7 @@ require "pg_ha_migrations/allowed_versions"
64
83
  require "pg_ha_migrations/railtie"
65
84
  require "pg_ha_migrations/hacks/disable_ddl_transaction"
66
85
  require "pg_ha_migrations/hacks/cleanup_unnecessary_output"
86
+ require "pg_ha_migrations/hacks/add_index_on_only"
67
87
 
68
88
  module PgHaMigrations::AutoIncluder
69
89
  def inherited(klass)
@@ -32,12 +32,12 @@ Gem::Specification.new do |spec|
32
32
  spec.add_development_dependency "rake", ">= 12.3.3"
33
33
  spec.add_development_dependency "rspec", "~> 3.0"
34
34
  spec.add_development_dependency "pg"
35
- spec.add_development_dependency "db-query-matchers", "~> 0.11.0"
35
+ spec.add_development_dependency "db-query-matchers", "~> 0.12.0"
36
36
  spec.add_development_dependency "pry"
37
37
  spec.add_development_dependency "pry-byebug"
38
38
  spec.add_development_dependency "appraisal", "~> 2.2.0"
39
39
 
40
- spec.add_dependency "rails", ">= 6.1", "< 7.1"
40
+ spec.add_dependency "rails", ">= 6.1", "< 7.2"
41
41
  spec.add_dependency "relation_to_struct", ">= 1.5.1"
42
42
  spec.add_dependency "ruby2_keywords"
43
43
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pg_ha_migrations
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.7.0
4
+ version: 1.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - celeen
@@ -14,7 +14,7 @@ authors:
14
14
  autorequire:
15
15
  bindir: exe
16
16
  cert_chain: []
17
- date: 2023-08-10 00:00:00.000000000 Z
17
+ date: 2024-01-12 00:00:00.000000000 Z
18
18
  dependencies:
19
19
  - !ruby/object:Gem::Dependency
20
20
  name: rake
@@ -64,14 +64,14 @@ dependencies:
64
64
  requirements:
65
65
  - - "~>"
66
66
  - !ruby/object:Gem::Version
67
- version: 0.11.0
67
+ version: 0.12.0
68
68
  type: :development
69
69
  prerelease: false
70
70
  version_requirements: !ruby/object:Gem::Requirement
71
71
  requirements:
72
72
  - - "~>"
73
73
  - !ruby/object:Gem::Version
74
- version: 0.11.0
74
+ version: 0.12.0
75
75
  - !ruby/object:Gem::Dependency
76
76
  name: pry
77
77
  requirement: !ruby/object:Gem::Requirement
@@ -123,7 +123,7 @@ dependencies:
123
123
  version: '6.1'
124
124
  - - "<"
125
125
  - !ruby/object:Gem::Version
126
- version: '7.1'
126
+ version: '7.2'
127
127
  type: :runtime
128
128
  prerelease: false
129
129
  version_requirements: !ruby/object:Gem::Requirement
@@ -133,7 +133,7 @@ dependencies:
133
133
  version: '6.1'
134
134
  - - "<"
135
135
  - !ruby/object:Gem::Version
136
- version: '7.1'
136
+ version: '7.2'
137
137
  - !ruby/object:Gem::Dependency
138
138
  name: relation_to_struct
139
139
  requirement: !ruby/object:Gem::Requirement
@@ -188,15 +188,19 @@ files:
188
188
  - gemfiles/.bundle/config
189
189
  - gemfiles/rails_6.1.gemfile
190
190
  - gemfiles/rails_7.0.gemfile
191
+ - gemfiles/rails_7.1.gemfile
191
192
  - lib/pg_ha_migrations.rb
192
193
  - lib/pg_ha_migrations/allowed_versions.rb
193
194
  - lib/pg_ha_migrations/blocking_database_transactions.rb
194
195
  - lib/pg_ha_migrations/blocking_database_transactions_reporter.rb
195
196
  - lib/pg_ha_migrations/dependent_objects_checks.rb
197
+ - lib/pg_ha_migrations/hacks/add_index_on_only.rb
196
198
  - lib/pg_ha_migrations/hacks/cleanup_unnecessary_output.rb
197
199
  - lib/pg_ha_migrations/hacks/disable_ddl_transaction.rb
200
+ - lib/pg_ha_migrations/lock_mode.rb
198
201
  - lib/pg_ha_migrations/partman_config.rb
199
202
  - lib/pg_ha_migrations/railtie.rb
203
+ - lib/pg_ha_migrations/relation.rb
200
204
  - lib/pg_ha_migrations/safe_statements.rb
201
205
  - lib/pg_ha_migrations/unsafe_statements.rb
202
206
  - lib/pg_ha_migrations/version.rb