code0-zero_track 0.0.9 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +36 -14
- data/db/migrate/database_partitioning/20261024153522_create_partitioning_schema.rb +13 -0
- data/db/migrate/database_partitioning/20261024153725_create_partitioning_views.rb +68 -0
- data/lib/code0/zero_track/database/migration.rb +11 -0
- data/lib/code0/zero_track/database/migration_helpers/polymorphic_trigger_helpers.rb +78 -0
- data/lib/code0/zero_track/database/migration_helpers/table_partitioning.rb +23 -21
- data/lib/code0/zero_track/database/partitioning/engine.rb +25 -0
- data/lib/code0/zero_track/database/partitioning/partition_manager.rb +69 -0
- data/lib/code0/zero_track/database/partitioning/partitioned_table.rb +10 -0
- data/lib/code0/zero_track/database/partitioning/schema_cleaner.rb +39 -0
- data/lib/code0/zero_track/database/schema_cleaner.rb +3 -28
- data/lib/code0/zero_track/features/database_partitioning.rb +25 -0
- data/lib/code0/zero_track/railtie.rb +1 -4
- data/lib/code0/zero_track/version.rb +1 -1
- data/lib/code0/zero_track.rb +8 -1
- metadata +7 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 03f2043062610cd62befd64e4dfc9ec9711dc2152059a167d79d35e303a52ae4
|
|
4
|
+
data.tar.gz: da719e16658fc3f52766edabd6c0295f7bfad788b727e5d19cf6825f6be2746a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8bce9799e2716ea52eea6e84a78c404089d6a9d613ea1b6b8c93830aef30d5d33aace17627d0eaedc3eebd19c368ebbd904d79a48fc7fd9b91c4fc2bbafde6e6
|
|
7
|
+
data.tar.gz: 2c8eb1c633d91d3fd64016eb2c52ee98653152e9bda9f89f31d8932d2f02ed32799d5c1a5a6435a6aa7dffdda0d390d4f491b0bd8956ad0562c5f5b1a2f834f8
|
data/README.md
CHANGED
|
@@ -72,6 +72,21 @@ the `db/structure.sql`, this mode creates files in the `db/schema_migrations` di
|
|
|
72
72
|
PostgreSQL supports declarative table partitioning. The partition manager automates the
|
|
73
73
|
management of partitions without manual operations or extensions on the PostgreSQL server.
|
|
74
74
|
|
|
75
|
+
Table partitioning is an optional feature which has to be explicitly required.
|
|
76
|
+
There are multiple options to do this:
|
|
77
|
+
|
|
78
|
+
1. Add the `require` above the application class definition in `config/application.rb`:
|
|
79
|
+
|
|
80
|
+
```ruby
|
|
81
|
+
require 'code0/zero_track/features/database_partitioning'
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
2. Use the `require` option in the `Gemfile`
|
|
85
|
+
|
|
86
|
+
```ruby
|
|
87
|
+
gem 'code0-zero_track', require: %w[code0/zero_track/features/database_partitioning]
|
|
88
|
+
```
|
|
89
|
+
|
|
75
90
|
#### Configuration
|
|
76
91
|
|
|
77
92
|
```ruby
|
|
@@ -82,31 +97,37 @@ config.zero_track.db_partitioning.base_ar_class = 'ActiveRecord::Base' # default
|
|
|
82
97
|
- `dynamic_partition_schema`: The PostgreSQL schema where dynamic partitions are stored.
|
|
83
98
|
- `base_ar_class`: The ActiveRecord base class used for the internal partitioning models.
|
|
84
99
|
|
|
100
|
+
#### Schema and Views Setup
|
|
101
|
+
|
|
102
|
+
The partition manager depends on a few PostgreSQL views.
|
|
103
|
+
|
|
104
|
+
The migrations to create these views can be installed as following:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
$ bin/rails code0_zero_track_database_partitioning:install:migrations
|
|
108
|
+
$ bin/rails db:migrate
|
|
109
|
+
```
|
|
110
|
+
|
|
85
111
|
#### Migration Helpers
|
|
86
112
|
|
|
87
|
-
Include the migration helpers by inheriting from `Code0::ZeroTrack::Database::Migration[1.
|
|
88
|
-
appropriate version). The following
|
|
113
|
+
Include the migration helpers by inheriting from `Code0::ZeroTrack::Database::Migration[1.1]` (or the
|
|
114
|
+
appropriate version). The following method becomes available:
|
|
89
115
|
|
|
90
116
|
`create_partition_by_date_table(table_name, partition_column:, primary_key: nil, **options, &block)` creates a table
|
|
91
117
|
partitioned by range on the given column. By default, it adds a `bigserial` id column and sets up a composite
|
|
92
118
|
primary key of `(id, partition_column)`. If `primary_key` is provided (e.g. `primary_key: %i[date other_column]`),
|
|
93
119
|
the id column is omitted and the given columns are used as the primary key instead.
|
|
94
120
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
(`postgres_partitioned_tables`, `postgres_partitions`, `postgres_detached_partitions`) that the
|
|
100
|
-
partition manager uses to inspect existing partitions.
|
|
121
|
+
> The schema/view helpers `create_dynamic_partition_schema`, `drop_dynamic_partition_schema`,
|
|
122
|
+
> `create_partitioning_views` and `drop_partitioning_views` have been removed in migration version 1.1.
|
|
123
|
+
> The schema and views no longer have to be created in your own migration with these helpers and instead
|
|
124
|
+
> are created by migrations from the gem.
|
|
101
125
|
|
|
102
126
|
Example migration:
|
|
103
127
|
|
|
104
128
|
```ruby
|
|
105
|
-
class CreatePartitionedEvents < Code0::ZeroTrack::Database::Migration[1.
|
|
129
|
+
class CreatePartitionedEvents < Code0::ZeroTrack::Database::Migration[1.1]
|
|
106
130
|
def change
|
|
107
|
-
create_dynamic_partition_schema
|
|
108
|
-
create_partitioning_views
|
|
109
|
-
|
|
110
131
|
create_partition_by_date_table :events, partition_column: :created_at do |t|
|
|
111
132
|
t.text :name, null: false
|
|
112
133
|
t.timestamps_with_timezone null: false
|
|
@@ -115,8 +136,6 @@ class CreatePartitionedEvents < Code0::ZeroTrack::Database::Migration[1.0]
|
|
|
115
136
|
end
|
|
116
137
|
```
|
|
117
138
|
|
|
118
|
-
The schema and views only need to be created once before creating the first partitioned table.
|
|
119
|
-
|
|
120
139
|
Tables don't necessarily have to be created with the provided helper. The partition manager will
|
|
121
140
|
work as long as the model is correctly configured.
|
|
122
141
|
|
|
@@ -189,6 +208,9 @@ When tables have foreign key relationships, registration order and retention con
|
|
|
189
208
|
- A child table's `retain_for` must be less than or equal to the parent table's `retain_for`.
|
|
190
209
|
If a child retains partitions longer than its parent, dropping the parent partition will
|
|
191
210
|
fail because the child's foreign key still references it.
|
|
211
|
+
- Foreign keys between the partitioned tables must be declared with `drop_foreign_keys_on_detach`
|
|
212
|
+
on the model. Otherwise, the foreign key of a detached partition will block detaching partitions
|
|
213
|
+
of the referenced table.
|
|
192
214
|
|
|
193
215
|
#### Schema Cleaner Integration
|
|
194
216
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class CreatePartitioningSchema < Code0::ZeroTrack::Database::Migration[1.1]
|
|
4
|
+
def up
|
|
5
|
+
schema = quote_table_name(Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema)
|
|
6
|
+
execute "CREATE SCHEMA #{schema}"
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
def down
|
|
10
|
+
schema = quote_table_name(Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema)
|
|
11
|
+
execute "DROP SCHEMA #{schema}"
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class CreatePartitioningViews < Code0::ZeroTrack::Database::Migration[1.1]
|
|
4
|
+
def up
|
|
5
|
+
dynamic_schema = Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema
|
|
6
|
+
|
|
7
|
+
execute <<~SQL.squish
|
|
8
|
+
CREATE OR REPLACE VIEW postgres_partitioned_tables AS
|
|
9
|
+
SELECT c.oid::regclass::text AS identifier,
|
|
10
|
+
c.oid,
|
|
11
|
+
n.nspname AS schema,
|
|
12
|
+
c.relname AS name,
|
|
13
|
+
CASE p.partstrat
|
|
14
|
+
WHEN 'l' THEN 'list'
|
|
15
|
+
WHEN 'r' THEN 'range'
|
|
16
|
+
WHEN 'h' THEN 'hash'
|
|
17
|
+
END AS strategy,
|
|
18
|
+
pg_get_partkeydef(c.oid) AS partition_key
|
|
19
|
+
FROM pg_partitioned_table p
|
|
20
|
+
JOIN pg_class c ON c.oid = p.partrelid
|
|
21
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
22
|
+
WHERE n.nspname = current_schema();
|
|
23
|
+
SQL
|
|
24
|
+
|
|
25
|
+
execute <<~SQL.squish
|
|
26
|
+
CREATE OR REPLACE VIEW postgres_partitions AS
|
|
27
|
+
SELECT c.oid::regclass::text AS identifier,
|
|
28
|
+
c.oid,
|
|
29
|
+
n.nspname AS schema,
|
|
30
|
+
c.relname AS name,
|
|
31
|
+
i.inhparent::regclass::text AS parent_identifier,
|
|
32
|
+
pg_get_expr(c.relpartbound, c.oid) AS condition,
|
|
33
|
+
obj_description(c.oid) AS comment,
|
|
34
|
+
i.inhrelid IS NOT NULL AS attached
|
|
35
|
+
FROM pg_class c
|
|
36
|
+
LEFT JOIN pg_inherits i ON c.oid = i.inhrelid
|
|
37
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
38
|
+
WHERE c.relispartition
|
|
39
|
+
AND c.relkind = 'r'
|
|
40
|
+
AND n.nspname IN (current_schema(), #{quote(dynamic_schema)});
|
|
41
|
+
SQL
|
|
42
|
+
|
|
43
|
+
execute <<~SQL.squish
|
|
44
|
+
CREATE OR REPLACE VIEW postgres_detached_partitions AS
|
|
45
|
+
SELECT c.oid::regclass::text AS identifier,
|
|
46
|
+
c.oid,
|
|
47
|
+
n.nspname AS schema,
|
|
48
|
+
c.relname AS name,
|
|
49
|
+
obj_description(c.oid)::jsonb ->> 'table' AS parent_identifier,
|
|
50
|
+
(obj_description(c.oid)::jsonb ->> 'detached_at')::timestamptz AS detached_at
|
|
51
|
+
FROM pg_class c
|
|
52
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
53
|
+
WHERE c.relkind = 'r'
|
|
54
|
+
AND n.nspname = #{quote(dynamic_schema)}
|
|
55
|
+
AND NOT EXISTS (
|
|
56
|
+
SELECT 1 FROM pg_inherits WHERE inhrelid = c.oid
|
|
57
|
+
)
|
|
58
|
+
AND obj_description(c.oid)::jsonb ? 'table'
|
|
59
|
+
AND obj_description(c.oid)::jsonb ? 'detached_at';
|
|
60
|
+
SQL
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def down
|
|
64
|
+
execute 'DROP VIEW IF EXISTS postgres_detached_partitions'
|
|
65
|
+
execute 'DROP VIEW IF EXISTS postgres_partitions'
|
|
66
|
+
execute 'DROP VIEW IF EXISTS postgres_partitioned_tables'
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -12,6 +12,17 @@ module Code0
|
|
|
12
12
|
include Database::MigrationHelpers::RemoveColumnEnhancements
|
|
13
13
|
include Database::MigrationHelpers::TableEnhancements
|
|
14
14
|
include Database::MigrationHelpers::TablePartitioning
|
|
15
|
+
include Database::MigrationHelpers::TablePartitioning::V1_0
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
class V1_1 < ::ActiveRecord::Migration[7.1]
|
|
19
|
+
include Database::MigrationHelpers::AddColumnEnhancements
|
|
20
|
+
include Database::MigrationHelpers::ConstraintHelpers
|
|
21
|
+
include Database::MigrationHelpers::IndexHelpers
|
|
22
|
+
include Database::MigrationHelpers::RemoveColumnEnhancements
|
|
23
|
+
include Database::MigrationHelpers::TableEnhancements
|
|
24
|
+
include Database::MigrationHelpers::TablePartitioning
|
|
25
|
+
include Database::MigrationHelpers::PolymorphicTriggerHelpers
|
|
15
26
|
end
|
|
16
27
|
# rubocop:enable Naming/ClassAndModuleCamelCase
|
|
17
28
|
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'digest'
|
|
4
|
+
|
|
5
|
+
module Code0
|
|
6
|
+
module ZeroTrack
|
|
7
|
+
module Database
|
|
8
|
+
module MigrationHelpers
|
|
9
|
+
# Helpers for managing polymorphic cleanup triggers.
|
|
10
|
+
#
|
|
11
|
+
# Rails does not enforce referential integrity for polymorphic associations, so deleting a
|
|
12
|
+
# parent record leaves orphaned child rows behind. These helpers install a PostgreSQL trigger
|
|
13
|
+
# that deletes the dependent polymorphic rows whenever a parent row is deleted.
|
|
14
|
+
module PolymorphicTriggerHelpers
|
|
15
|
+
POLYMORPHIC_CLEANUP_FUNCTION_NAME = 'delete_polymorphic_records_v1'
|
|
16
|
+
POLYMORPHIC_CLEANUP_TRANSITION_TABLE = 'old_rows'
|
|
17
|
+
|
|
18
|
+
def create_polymorphic_cleanup_trigger(table, parent_table, parent_column, parent_class)
|
|
19
|
+
create_polymorphic_cleanup_function
|
|
20
|
+
|
|
21
|
+
trigger_name = polymorphic_cleanup_trigger_name(table, parent_table, parent_column)
|
|
22
|
+
|
|
23
|
+
execute(<<~SQL) # rubocop:disable Rails/SquishedSQLHeredocs -- we want to keep the formatting
|
|
24
|
+
CREATE TRIGGER #{quote_table_name(trigger_name)}
|
|
25
|
+
AFTER DELETE ON #{quote_table_name(parent_table)}
|
|
26
|
+
REFERENCING OLD TABLE AS #{POLYMORPHIC_CLEANUP_TRANSITION_TABLE}
|
|
27
|
+
FOR EACH STATEMENT
|
|
28
|
+
EXECUTE FUNCTION #{quote_table_name(POLYMORPHIC_CLEANUP_FUNCTION_NAME)}(
|
|
29
|
+
#{quote(table.to_s)},
|
|
30
|
+
#{quote(parent_column.to_s)},
|
|
31
|
+
#{quote(parent_class.to_s)}
|
|
32
|
+
);
|
|
33
|
+
SQL
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def drop_polymorphic_cleanup_trigger(table, parent_table, parent_column)
|
|
37
|
+
trigger_name = polymorphic_cleanup_trigger_name(table, parent_table, parent_column)
|
|
38
|
+
|
|
39
|
+
execute(<<~SQL.squish)
|
|
40
|
+
DROP TRIGGER IF EXISTS #{quote_table_name(trigger_name)} ON #{quote_table_name(parent_table)};
|
|
41
|
+
SQL
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def create_polymorphic_cleanup_function
|
|
45
|
+
execute(<<~SQL) # rubocop:disable Rails/SquishedSQLHeredocs -- we want to keep the formatting
|
|
46
|
+
CREATE OR REPLACE FUNCTION #{quote_table_name(POLYMORPHIC_CLEANUP_FUNCTION_NAME)}()
|
|
47
|
+
RETURNS TRIGGER AS $$
|
|
48
|
+
BEGIN
|
|
49
|
+
EXECUTE format(
|
|
50
|
+
'DELETE FROM %I WHERE %I = $1 AND %I IN (SELECT id FROM #{POLYMORPHIC_CLEANUP_TRANSITION_TABLE})',
|
|
51
|
+
TG_ARGV[0],
|
|
52
|
+
TG_ARGV[1] || '_type',
|
|
53
|
+
TG_ARGV[1] || '_id'
|
|
54
|
+
) USING TG_ARGV[2];
|
|
55
|
+
RETURN NULL;
|
|
56
|
+
END;
|
|
57
|
+
$$ LANGUAGE plpgsql;
|
|
58
|
+
SQL
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# This method should only be used to clean up the function once no trigger uses it anymore.
|
|
62
|
+
def drop_polymorphic_cleanup_function
|
|
63
|
+
execute(<<~SQL.squish)
|
|
64
|
+
DROP FUNCTION IF EXISTS #{quote_table_name(POLYMORPHIC_CLEANUP_FUNCTION_NAME)}();
|
|
65
|
+
SQL
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def polymorphic_cleanup_trigger_name(table, parent_table, parent_column)
|
|
69
|
+
identifier = "#{parent_table}_#{parent_column}_#{table}"
|
|
70
|
+
hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10)
|
|
71
|
+
|
|
72
|
+
"trigger_delete_polymorphic_#{hashed_identifier}"
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -27,20 +27,21 @@ module Code0
|
|
|
27
27
|
end
|
|
28
28
|
end
|
|
29
29
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
30
|
+
module V1_0 # rubocop:disable Naming/ClassAndModuleCamelCase
|
|
31
|
+
def create_dynamic_partition_schema
|
|
32
|
+
schema = quote_table_name(Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema)
|
|
33
|
+
execute "CREATE SCHEMA #{schema}"
|
|
34
|
+
end
|
|
34
35
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
36
|
+
def drop_dynamic_partition_schema
|
|
37
|
+
schema = quote_table_name(Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema)
|
|
38
|
+
execute "DROP SCHEMA #{schema}"
|
|
39
|
+
end
|
|
39
40
|
|
|
40
|
-
|
|
41
|
-
|
|
41
|
+
def create_partitioning_views
|
|
42
|
+
dynamic_schema = Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema
|
|
42
43
|
|
|
43
|
-
|
|
44
|
+
execute <<-SQL.squish
|
|
44
45
|
CREATE OR REPLACE VIEW postgres_partitioned_tables AS
|
|
45
46
|
SELECT c.oid::regclass::text AS identifier,
|
|
46
47
|
c.oid,
|
|
@@ -56,9 +57,9 @@ module Code0
|
|
|
56
57
|
JOIN pg_class c ON c.oid = p.partrelid
|
|
57
58
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
58
59
|
WHERE n.nspname = current_schema();
|
|
59
|
-
|
|
60
|
+
SQL
|
|
60
61
|
|
|
61
|
-
|
|
62
|
+
execute <<-SQL.squish
|
|
62
63
|
CREATE OR REPLACE VIEW postgres_partitions AS
|
|
63
64
|
SELECT c.oid::regclass::text AS identifier,
|
|
64
65
|
c.oid,
|
|
@@ -74,9 +75,9 @@ module Code0
|
|
|
74
75
|
WHERE c.relispartition
|
|
75
76
|
AND c.relkind = 'r'
|
|
76
77
|
AND n.nspname IN (current_schema(), #{quote(dynamic_schema)});
|
|
77
|
-
|
|
78
|
+
SQL
|
|
78
79
|
|
|
79
|
-
|
|
80
|
+
execute <<-SQL.squish
|
|
80
81
|
CREATE OR REPLACE VIEW postgres_detached_partitions AS
|
|
81
82
|
SELECT c.oid::regclass::text AS identifier,
|
|
82
83
|
c.oid,
|
|
@@ -93,13 +94,14 @@ module Code0
|
|
|
93
94
|
)
|
|
94
95
|
AND obj_description(c.oid)::jsonb ? 'table'
|
|
95
96
|
AND obj_description(c.oid)::jsonb ? 'detached_at';
|
|
96
|
-
|
|
97
|
-
|
|
97
|
+
SQL
|
|
98
|
+
end
|
|
98
99
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
100
|
+
def drop_partitioning_views
|
|
101
|
+
execute 'DROP VIEW IF EXISTS postgres_detached_partitions'
|
|
102
|
+
execute 'DROP VIEW IF EXISTS postgres_partitions'
|
|
103
|
+
execute 'DROP VIEW IF EXISTS postgres_partitioned_tables'
|
|
104
|
+
end
|
|
103
105
|
end
|
|
104
106
|
end
|
|
105
107
|
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Code0
|
|
4
|
+
module ZeroTrack
|
|
5
|
+
module Database
|
|
6
|
+
module Partitioning
|
|
7
|
+
class Engine < Rails::Engine
|
|
8
|
+
isolate_namespace Partitioning
|
|
9
|
+
|
|
10
|
+
config.zero_track.db_partitioning = ActiveSupport::OrderedOptions.new
|
|
11
|
+
config.zero_track.db_partitioning.dynamic_partition_schema = 'partitions_dynamic'
|
|
12
|
+
config.zero_track.db_partitioning.base_ar_class = 'ActiveRecord::Base'
|
|
13
|
+
|
|
14
|
+
initializer 'code0-zero_track.database_partitioning.paths' do
|
|
15
|
+
paths['db/migrate'] = 'db/migrate/database_partitioning'
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
initializer 'code0-zero_track.database_partitioning.register_schema_cleaner' do
|
|
19
|
+
config.zero_track.active_record.schema_cleaner_callbacks << SchemaCleaner.new
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -64,6 +64,8 @@ module Code0
|
|
|
64
64
|
model.partitioning_strategy.partitions_to_detach.each do |partition|
|
|
65
65
|
detach_partition(partition, connection)
|
|
66
66
|
end
|
|
67
|
+
|
|
68
|
+
drop_declared_foreign_keys_from_detached_partitions(connection)
|
|
67
69
|
end
|
|
68
70
|
end
|
|
69
71
|
|
|
@@ -109,6 +111,73 @@ module Code0
|
|
|
109
111
|
)
|
|
110
112
|
end
|
|
111
113
|
|
|
114
|
+
def drop_declared_foreign_keys_from_detached_partitions(connection)
|
|
115
|
+
constraint_names = model.foreign_keys_to_drop_on_detach || []
|
|
116
|
+
return if constraint_names.empty?
|
|
117
|
+
|
|
118
|
+
warn_about_unknown_foreign_keys(constraint_names, connection)
|
|
119
|
+
|
|
120
|
+
detached_foreign_keys(constraint_names, connection).each do |qualified_table, constraint_name|
|
|
121
|
+
connection.execute(
|
|
122
|
+
"ALTER TABLE #{qualified_table} " \
|
|
123
|
+
"DROP CONSTRAINT #{connection.quote_column_name(constraint_name)}"
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
logger.info(
|
|
127
|
+
message: 'Dropped foreign key from detached partition',
|
|
128
|
+
table_name: model.table_name,
|
|
129
|
+
partition_name: qualified_table,
|
|
130
|
+
constraint_name: constraint_name
|
|
131
|
+
)
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# We check the parent table because we drop the FK from the partition
|
|
136
|
+
# when detaching. A missing FK on the detached partition is an expected state
|
|
137
|
+
def warn_about_unknown_foreign_keys(constraint_names, connection)
|
|
138
|
+
existing = parent_foreign_key_names(connection)
|
|
139
|
+
unknown = constraint_names - existing
|
|
140
|
+
return if unknown.empty?
|
|
141
|
+
|
|
142
|
+
logger.warn(
|
|
143
|
+
message: 'Configured foreign keys to drop on detach do not exist on the partitioned table',
|
|
144
|
+
table_name: model.table_name,
|
|
145
|
+
constraint_names: unknown
|
|
146
|
+
)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def parent_foreign_key_names(connection)
|
|
150
|
+
connection.select_values(<<~SQL.squish)
|
|
151
|
+
SELECT con.conname
|
|
152
|
+
FROM pg_catalog.pg_constraint con
|
|
153
|
+
WHERE con.contype = 'f'
|
|
154
|
+
AND con.conrelid = #{connection.quote(model.table_name)}::regclass
|
|
155
|
+
SQL
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def detached_foreign_keys(constraint_names, connection)
|
|
159
|
+
schema = Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema
|
|
160
|
+
quoted_names = constraint_names.map { |name| connection.quote(name) }.join(', ')
|
|
161
|
+
|
|
162
|
+
rows = connection.select_rows(<<~SQL.squish)
|
|
163
|
+
SELECT (quote_ident(n.nspname) || '.' || quote_ident(c.relname)) AS qualified_table,
|
|
164
|
+
con.conname AS constraint_name
|
|
165
|
+
FROM pg_catalog.pg_constraint con
|
|
166
|
+
JOIN pg_catalog.pg_class c ON c.oid = con.conrelid
|
|
167
|
+
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
|
168
|
+
WHERE con.contype = 'f'
|
|
169
|
+
AND con.conname IN (#{quoted_names})
|
|
170
|
+
AND n.nspname = #{connection.quote(schema)}
|
|
171
|
+
AND c.relkind = 'r'
|
|
172
|
+
AND NOT c.relispartition
|
|
173
|
+
AND NOT EXISTS (
|
|
174
|
+
SELECT 1 FROM pg_catalog.pg_inherits i WHERE i.inhrelid = c.oid
|
|
175
|
+
)
|
|
176
|
+
SQL
|
|
177
|
+
|
|
178
|
+
rows.map { |qualified_table, constraint_name| [qualified_table, constraint_name] }
|
|
179
|
+
end
|
|
180
|
+
|
|
112
181
|
def drop_partition(detached_partition, connection)
|
|
113
182
|
schema_name = connection.quote_table_name(detached_partition.schema)
|
|
114
183
|
partition_name = connection.quote_table_name(detached_partition.name)
|
|
@@ -25,6 +25,16 @@ module Code0
|
|
|
25
25
|
|
|
26
26
|
@partitioning_strategy = strategy_class.new(self, column, **kwargs)
|
|
27
27
|
end
|
|
28
|
+
|
|
29
|
+
def drop_foreign_keys_on_detach(*constraint_names)
|
|
30
|
+
@foreign_keys_to_drop_on_detach ||= []
|
|
31
|
+
@foreign_keys_to_drop_on_detach.concat(constraint_names.flatten.map(&:to_s))
|
|
32
|
+
@foreign_keys_to_drop_on_detach
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def foreign_keys_to_drop_on_detach
|
|
36
|
+
@foreign_keys_to_drop_on_detach || []
|
|
37
|
+
end
|
|
28
38
|
end
|
|
29
39
|
end
|
|
30
40
|
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Code0
|
|
4
|
+
module ZeroTrack
|
|
5
|
+
module Database
|
|
6
|
+
module Partitioning
|
|
7
|
+
class SchemaCleaner
|
|
8
|
+
def clean!(structure)
|
|
9
|
+
schema = Regexp.escape(dynamic_partition_schema)
|
|
10
|
+
|
|
11
|
+
# Remove CREATE TABLE <schema>.<partition> (...);
|
|
12
|
+
structure.gsub!(/^CREATE TABLE #{schema}\.\S+\s*\(.*?\);\n/m, '')
|
|
13
|
+
|
|
14
|
+
# Remove ALTER TABLE ... ATTACH PARTITION <schema>.<partition> ...;
|
|
15
|
+
structure.gsub!(/^ALTER TABLE .+ ATTACH PARTITION #{schema}\.\S+.*?;\n/, '')
|
|
16
|
+
|
|
17
|
+
# Remove ALTER TABLE ONLY <schema>.<partition> ...;
|
|
18
|
+
structure.gsub!(/^ALTER TABLE ONLY #{schema}\.\S+\n.*?;\n/m, '')
|
|
19
|
+
|
|
20
|
+
# Remove CREATE [UNIQUE] INDEX ... ON <schema>.<partition> ...;
|
|
21
|
+
structure.gsub!(/^CREATE (?:UNIQUE )?INDEX \S+ ON #{schema}\.\S+.*?;\n/m, '')
|
|
22
|
+
|
|
23
|
+
# Remove ALTER INDEX ... ATTACH PARTITION <schema>.<partition>;
|
|
24
|
+
structure.gsub!(/^ALTER INDEX \S+ ATTACH PARTITION #{schema}\.\S+;\n/, '')
|
|
25
|
+
|
|
26
|
+
# Remove COMMENT ON TABLE <schema>.<partition> IS '<comment>';
|
|
27
|
+
structure.gsub!(/^COMMENT ON TABLE #{schema}\.\S+ IS .*;\n/, '')
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def dynamic_partition_schema
|
|
33
|
+
Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -38,9 +38,9 @@ module Code0
|
|
|
38
38
|
'CREATE EXTENSION IF NOT EXISTS \1;'
|
|
39
39
|
)
|
|
40
40
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
41
|
+
Rails.application.config.zero_track.active_record.schema_cleaner_callbacks.each do |cleaner|
|
|
42
|
+
cleaner.clean!(structure)
|
|
43
|
+
end
|
|
44
44
|
|
|
45
45
|
structure.gsub!(/\n{3,}/, "\n\n")
|
|
46
46
|
|
|
@@ -49,31 +49,6 @@ module Code0
|
|
|
49
49
|
|
|
50
50
|
nil
|
|
51
51
|
end
|
|
52
|
-
|
|
53
|
-
private
|
|
54
|
-
|
|
55
|
-
def dynamic_partition_schema
|
|
56
|
-
Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
def remove_dynamic_partitions!(structure)
|
|
60
|
-
schema = Regexp.escape(dynamic_partition_schema)
|
|
61
|
-
|
|
62
|
-
# Remove CREATE TABLE <schema>.<partition> (...);
|
|
63
|
-
structure.gsub!(/^CREATE TABLE #{schema}\.\S+\s*\(.*?\);\n/m, '')
|
|
64
|
-
|
|
65
|
-
# Remove ALTER TABLE ... ATTACH PARTITION <schema>.<partition> ...;
|
|
66
|
-
structure.gsub!(/^ALTER TABLE .+ ATTACH PARTITION #{schema}\.\S+.*?;\n/, '')
|
|
67
|
-
|
|
68
|
-
# Remove ALTER TABLE ONLY <schema>.<partition> ...;
|
|
69
|
-
structure.gsub!(/^ALTER TABLE ONLY #{schema}\.\S+\n.*?;\n/m, '')
|
|
70
|
-
|
|
71
|
-
# Remove CREATE [UNIQUE] INDEX ... ON <schema>.<partition> ...;
|
|
72
|
-
structure.gsub!(/^CREATE (?:UNIQUE )?INDEX \S+ ON #{schema}\.\S+.*?;\n/m, '')
|
|
73
|
-
|
|
74
|
-
# Remove ALTER INDEX ... ATTACH PARTITION <schema>.<partition>;
|
|
75
|
-
structure.gsub!(/^ALTER INDEX \S+ ATTACH PARTITION #{schema}\.\S+;\n/, '')
|
|
76
|
-
end
|
|
77
52
|
end
|
|
78
53
|
end
|
|
79
54
|
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'code0/zero_track'
|
|
4
|
+
|
|
5
|
+
loader = Zeitwerk::Loader.new
|
|
6
|
+
loader.tag = 'code0-zero_track/database_partitioning'
|
|
7
|
+
loader.inflector = Zeitwerk::GemInflector.new(File.join(__dir__, '../..'))
|
|
8
|
+
|
|
9
|
+
module Code0
|
|
10
|
+
module ZeroTrack
|
|
11
|
+
module Database
|
|
12
|
+
module Partitioning
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
loader.push_dir(
|
|
19
|
+
File.expand_path(File.join(__dir__, '../database/partitioning')),
|
|
20
|
+
namespace: Code0::ZeroTrack::Database::Partitioning
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
loader.setup
|
|
24
|
+
|
|
25
|
+
Code0::ZeroTrack::Database::Partitioning::Engine # eager load the engine
|
|
@@ -9,10 +9,7 @@ module Code0
|
|
|
9
9
|
config.zero_track.active_record.timestamps = false
|
|
10
10
|
config.zero_track.active_record.schema_migrations = false
|
|
11
11
|
config.zero_track.active_record.schema_cleaner = false
|
|
12
|
-
|
|
13
|
-
config.zero_track.db_partitioning = ActiveSupport::OrderedOptions.new
|
|
14
|
-
config.zero_track.db_partitioning.dynamic_partition_schema = 'partitions_dynamic'
|
|
15
|
-
config.zero_track.db_partitioning.base_ar_class = 'ActiveRecord::Base'
|
|
12
|
+
config.zero_track.active_record.schema_cleaner_callbacks = []
|
|
16
13
|
|
|
17
14
|
rake_tasks do
|
|
18
15
|
path = File.expand_path(__dir__)
|
data/lib/code0/zero_track.rb
CHANGED
|
@@ -5,9 +5,16 @@ require 'rails/railtie'
|
|
|
5
5
|
|
|
6
6
|
require 'zeitwerk'
|
|
7
7
|
loader = Zeitwerk::Loader.new
|
|
8
|
-
loader.tag =
|
|
8
|
+
loader.tag = 'code0-zero_track'
|
|
9
9
|
loader.inflector = Zeitwerk::GemInflector.new(__FILE__)
|
|
10
10
|
loader.push_dir(File.expand_path(File.join(__dir__, '..')))
|
|
11
|
+
|
|
12
|
+
# ignore feature entrypoints
|
|
13
|
+
loader.ignore("#{__dir__}/zero_track/features")
|
|
14
|
+
|
|
15
|
+
# database partitioning is a separate feature which needs to be explicitly required
|
|
16
|
+
loader.ignore("#{__dir__}/zero_track/database/partitioning")
|
|
17
|
+
|
|
11
18
|
loader.setup
|
|
12
19
|
|
|
13
20
|
module Code0
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: code0-zero_track
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0
|
|
4
|
+
version: 0.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Niklas van Schrick
|
|
@@ -158,6 +158,8 @@ files:
|
|
|
158
158
|
- LICENSE
|
|
159
159
|
- README.md
|
|
160
160
|
- Rakefile
|
|
161
|
+
- db/migrate/database_partitioning/20261024153522_create_partitioning_schema.rb
|
|
162
|
+
- db/migrate/database_partitioning/20261024153725_create_partitioning_views.rb
|
|
161
163
|
- lib/code0/zero_track.rb
|
|
162
164
|
- lib/code0/zero_track/context.rb
|
|
163
165
|
- lib/code0/zero_track/database/column_methods.rb
|
|
@@ -165,14 +167,17 @@ files:
|
|
|
165
167
|
- lib/code0/zero_track/database/migration_helpers/add_column_enhancements.rb
|
|
166
168
|
- lib/code0/zero_track/database/migration_helpers/constraint_helpers.rb
|
|
167
169
|
- lib/code0/zero_track/database/migration_helpers/index_helpers.rb
|
|
170
|
+
- lib/code0/zero_track/database/migration_helpers/polymorphic_trigger_helpers.rb
|
|
168
171
|
- lib/code0/zero_track/database/migration_helpers/remove_column_enhancements.rb
|
|
169
172
|
- lib/code0/zero_track/database/migration_helpers/table_enhancements.rb
|
|
170
173
|
- lib/code0/zero_track/database/migration_helpers/table_partitioning.rb
|
|
174
|
+
- lib/code0/zero_track/database/partitioning/engine.rb
|
|
171
175
|
- lib/code0/zero_track/database/partitioning/partition_manager.rb
|
|
172
176
|
- lib/code0/zero_track/database/partitioning/partitioned_table.rb
|
|
173
177
|
- lib/code0/zero_track/database/partitioning/postgres_detached_partition.rb
|
|
174
178
|
- lib/code0/zero_track/database/partitioning/postgres_partition.rb
|
|
175
179
|
- lib/code0/zero_track/database/partitioning/postgres_partitioned_table.rb
|
|
180
|
+
- lib/code0/zero_track/database/partitioning/schema_cleaner.rb
|
|
176
181
|
- lib/code0/zero_track/database/partitioning/strategy/base.rb
|
|
177
182
|
- lib/code0/zero_track/database/partitioning/strategy/daily.rb
|
|
178
183
|
- lib/code0/zero_track/database/partitioning/strategy/monthly.rb
|
|
@@ -185,6 +190,7 @@ files:
|
|
|
185
190
|
- lib/code0/zero_track/database/schema_migrations/context.rb
|
|
186
191
|
- lib/code0/zero_track/database/schema_migrations/formatter.rb
|
|
187
192
|
- lib/code0/zero_track/database/schema_migrations/migrations.rb
|
|
193
|
+
- lib/code0/zero_track/features/database_partitioning.rb
|
|
188
194
|
- lib/code0/zero_track/injectors/active_record_schema_migrations.rb
|
|
189
195
|
- lib/code0/zero_track/injectors/active_record_timestamps.rb
|
|
190
196
|
- lib/code0/zero_track/loggable.rb
|