partitioned 0.8.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +85 -36
- data/Rakefile +3 -0
- data/examples/README +46 -18
- data/lib/monkey_patch_activerecord.rb +14 -8
- data/lib/monkey_patch_postgres.rb +46 -13
- data/lib/partitioned/active_record_overrides.rb +13 -5
- data/lib/partitioned/bulk_methods_mixin.rb +91 -146
- data/lib/partitioned/by_created_at.rb +3 -1
- data/lib/partitioned/by_foreign_key.rb +5 -0
- data/lib/partitioned/by_id.rb +10 -4
- data/lib/partitioned/by_integer_field.rb +9 -0
- data/lib/partitioned/by_monthly_time_field.rb +8 -1
- data/lib/partitioned/by_time_field.rb +16 -8
- data/lib/partitioned/by_weekly_time_field.rb +6 -3
- data/lib/partitioned/multi_level/configurator/data.rb +1 -0
- data/lib/partitioned/multi_level/configurator/dsl.rb +11 -0
- data/lib/partitioned/multi_level/configurator/reader.rb +18 -0
- data/lib/partitioned/multi_level/partition_manager.rb +13 -4
- data/lib/partitioned/multi_level.rb +3 -1
- data/lib/partitioned/partitioned_base/configurator/data.rb +10 -1
- data/lib/partitioned/partitioned_base/configurator/dsl.rb +20 -15
- data/lib/partitioned/partitioned_base/configurator/reader.rb +3 -0
- data/lib/partitioned/partitioned_base/configurator.rb +4 -0
- data/lib/partitioned/partitioned_base/partition_manager.rb +17 -15
- data/lib/partitioned/partitioned_base/sql_adapter.rb +25 -23
- data/lib/partitioned/partitioned_base.rb +112 -41
- data/lib/partitioned/version.rb +2 -1
- data/partitioned.gemspec +3 -2
- metadata +68 -73
@@ -4,22 +4,22 @@ module Partitioned
|
|
4
4
|
#
|
5
5
|
# The Domain Specific Language manager for configuring partitioning.
|
6
6
|
#
|
7
|
-
# example:
|
7
|
+
# @example of use:
|
8
8
|
#
|
9
|
-
#
|
10
|
-
#
|
11
|
-
#
|
12
|
-
#
|
9
|
+
# class Employee < Partitioned::ByCreatedAt
|
10
|
+
# partitioned do |partition|
|
11
|
+
# partition.index :id, :unique => true
|
12
|
+
# partition.foreign_key :company_id
|
13
|
+
# end
|
13
14
|
# end
|
14
|
-
# end
|
15
15
|
#
|
16
|
-
#
|
16
|
+
# in the above example, block:
|
17
17
|
#
|
18
|
-
#
|
19
|
-
#
|
20
|
-
#
|
18
|
+
# partitioned do |partition|
|
19
|
+
# ...
|
20
|
+
# end
|
21
21
|
#
|
22
|
-
#
|
22
|
+
# Scopes a set of "partition directives". The directives are accessed via the block parameter 'partition'
|
23
23
|
#
|
24
24
|
# Directives parameters have two forms, a canonical form which takes a set of parameters
|
25
25
|
# and a dynamic form which takes a single parameter which is either a string that should be interpolated or
|
@@ -36,10 +36,11 @@ module Partitioned
|
|
36
36
|
# end
|
37
37
|
#
|
38
38
|
# partitioned do |partition|
|
39
|
-
# partition.on lambda{|model| return model.partition_time_field}
|
39
|
+
# partition.on lambda{ |model| return model.partition_time_field}
|
40
40
|
# partition.constraint lambda{|model,time_field_value|
|
41
|
-
#
|
42
|
-
#
|
41
|
+
# return "#{model.partition_time_field} >= '#{time_field_value.strftime}' and
|
42
|
+
# #{model.partition_time_field} < '#{(time_field_value + 1.day).strftime}'"
|
43
|
+
# }
|
43
44
|
# end
|
44
45
|
# end
|
45
46
|
#
|
@@ -134,6 +135,9 @@ module Partitioned
|
|
134
135
|
# resolve to FavoriteEmployee.index_field and be :baz as expected.
|
135
136
|
#
|
136
137
|
class Dsl
|
138
|
+
#
|
139
|
+
# raised when a partitioned DSL directive's parameters are considered invalid
|
140
|
+
#
|
137
141
|
class InvalidConfiguratorDirectiveValue < StandardError
|
138
142
|
def initialize(model, table_name, directive, value, explanation)
|
139
143
|
super("#{model.name} [#{table_name}] invalid value '#{value}' for partitioned directive '#{directive}'. #{explanation}")
|
@@ -254,7 +258,8 @@ module Partitioned
|
|
254
258
|
if referencing_field.is_a? Proc
|
255
259
|
data.foreign_keys << referencing_field
|
256
260
|
else
|
257
|
-
data.foreign_keys << Partitioned::PartitionedBase::Configurator::Data::ForeignKey.
|
261
|
+
data.foreign_keys << Partitioned::PartitionedBase::Configurator::Data::ForeignKey.
|
262
|
+
new(referencing_field, referenced_table, referenced_field)
|
258
263
|
end
|
259
264
|
end
|
260
265
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
1
3
|
module Partitioned
|
2
4
|
class PartitionedBase
|
3
5
|
#
|
@@ -13,7 +15,7 @@ module Partitioned
|
|
13
15
|
end
|
14
16
|
|
15
17
|
#
|
16
|
-
#
|
18
|
+
# Drop partitions that are no longer necessary.
|
17
19
|
# uses #old_partition_key_values_set as the list of
|
18
20
|
# partitions to remove.
|
19
21
|
#
|
@@ -24,7 +26,7 @@ module Partitioned
|
|
24
26
|
end
|
25
27
|
|
26
28
|
#
|
27
|
-
#
|
29
|
+
# Create partitions that are needed (probably to handle data that
|
28
30
|
# will be inserted into the database within the next few weeks).
|
29
31
|
# uses #new_partition_key_value_set to determine the key values
|
30
32
|
# for the specific child tables to create.
|
@@ -36,7 +38,7 @@ module Partitioned
|
|
36
38
|
end
|
37
39
|
|
38
40
|
#
|
39
|
-
#
|
41
|
+
# Create any partition tables from a list. the partition tables must
|
40
42
|
# not already exist and its schema must already exist.
|
41
43
|
#
|
42
44
|
def create_new_partition_tables(enumerable)
|
@@ -46,7 +48,7 @@ module Partitioned
|
|
46
48
|
end
|
47
49
|
|
48
50
|
#
|
49
|
-
#
|
51
|
+
# The once called function to prepare a parent table for partitioning as well
|
50
52
|
# as create the schema that the child tables will be placed in.
|
51
53
|
#
|
52
54
|
def create_infrastructure
|
@@ -56,41 +58,41 @@ module Partitioned
|
|
56
58
|
|
57
59
|
protected
|
58
60
|
#
|
59
|
-
#
|
61
|
+
# An array of key values (each key value is an array of keys) that represent
|
60
62
|
# the child partitions that should be created.
|
61
63
|
#
|
62
|
-
#
|
63
|
-
# the database with new soon-to-be needed child tables
|
64
|
+
# Used by #create_new_partitions and generally called once a day to update
|
65
|
+
# the database with new soon-to-be needed child tables.
|
64
66
|
#
|
65
|
-
#
|
67
|
+
# Typically overridden by the concrete class as this is pure business logic.
|
66
68
|
#
|
67
69
|
def new_partition_key_values_set
|
68
70
|
[]
|
69
71
|
end
|
70
72
|
|
71
73
|
#
|
72
|
-
#
|
74
|
+
# An array of key values (each key value is an array of keys) that represent
|
73
75
|
# the child partitions that should be dropped because they are no longer needed.
|
74
76
|
#
|
75
|
-
#
|
76
|
-
# unneeded child tables
|
77
|
+
# Used by #drop_old_partitions and generally called once a day to clean up
|
78
|
+
# unneeded child tables.
|
77
79
|
#
|
78
|
-
#
|
80
|
+
# Typically overridden by the concrete class as this is pure business logic.
|
79
81
|
#
|
80
82
|
def old_partition_key_values_set
|
81
83
|
[]
|
82
84
|
end
|
83
85
|
|
84
86
|
#
|
85
|
-
#
|
86
|
-
# the key value(s) of its check constraint columns
|
87
|
+
# Remove a specific partition from the database given
|
88
|
+
# the key value(s) of its check constraint columns.
|
87
89
|
#
|
88
90
|
def drop_old_partition(*partition_key_values)
|
89
91
|
drop_partition_table(*partition_key_values)
|
90
92
|
end
|
91
93
|
|
92
94
|
#
|
93
|
-
#
|
95
|
+
# Create a specific child table that does not currently
|
94
96
|
# exist and whose schema (the schema that the table exists in)
|
95
97
|
# also already exists (#create_infrastructure is designed to
|
96
98
|
# create this).
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
1
3
|
module Partitioned
|
2
4
|
class PartitionedBase
|
3
5
|
#
|
@@ -12,7 +14,7 @@ module Partitioned
|
|
12
14
|
end
|
13
15
|
|
14
16
|
#
|
15
|
-
#
|
17
|
+
# Ensure our function for warning about improper partition usage is in place.
|
16
18
|
#
|
17
19
|
# Name: always_fail_on_insert(text); Type: FUNCTION; Schema: public
|
18
20
|
#
|
@@ -43,7 +45,7 @@ module Partitioned
|
|
43
45
|
end
|
44
46
|
|
45
47
|
#
|
46
|
-
#
|
48
|
+
# Does a specific child partition exist.
|
47
49
|
#
|
48
50
|
def partition_exists?(*partition_key_values)
|
49
51
|
return find(:first,
|
@@ -56,22 +58,22 @@ module Partitioned
|
|
56
58
|
end
|
57
59
|
|
58
60
|
#
|
59
|
-
#
|
61
|
+
# Returns an array of partition table names from last to first limited to
|
60
62
|
# the number of entries requested by its first parameter.
|
61
63
|
#
|
62
|
-
#
|
64
|
+
# The magic here is in the overridden method "last_n_partitions_order_by_clause"
|
63
65
|
# which is designed to order a list of partition table names (table names without
|
64
66
|
# their schema name) from last to first.
|
65
67
|
#
|
66
|
-
#
|
68
|
+
# If the child table names are the format "pYYYYMMDD" where YYYY is a four digit year, MM is
|
67
69
|
# a month number and DD is a day number, you would use the following to order from last to
|
68
70
|
# first:
|
69
71
|
# tablename desc
|
70
72
|
#
|
71
|
-
#
|
73
|
+
# For child table names of the format "pXXXX" where XXXX is a number, you may want something like:
|
72
74
|
# substring(tablename, 2)::integer desc
|
73
75
|
#
|
74
|
-
#
|
76
|
+
# For clarity, the sql executed is:
|
75
77
|
# select tablename from pg_tables where schemaname = $1 order by $2 limit $3
|
76
78
|
# where:
|
77
79
|
# $1 = the name of schema (foos_partitions)
|
@@ -88,18 +90,18 @@ module Partitioned
|
|
88
90
|
end
|
89
91
|
|
90
92
|
#
|
91
|
-
#
|
93
|
+
# Override this or order the tables from last (greatest value? greatest date?) to first.
|
92
94
|
#
|
93
95
|
def last_n_partitions_order_by_clause
|
94
96
|
return configurator.last_n_partitions_order_by_clause
|
95
97
|
end
|
96
98
|
|
97
99
|
#
|
98
|
-
#
|
100
|
+
# Used to create the parent table rule to ensure.
|
99
101
|
#
|
100
|
-
#
|
102
|
+
# This will cause an error on attempt to insert into the parent table.
|
101
103
|
#
|
102
|
-
#
|
104
|
+
# We want all records to exist in one of the child tables so the
|
103
105
|
# query planner can optimize access to the records.
|
104
106
|
#
|
105
107
|
def add_parent_table_rules(*partition_key_values)
|
@@ -118,14 +120,14 @@ module Partitioned
|
|
118
120
|
end
|
119
121
|
|
120
122
|
#
|
121
|
-
#
|
123
|
+
# The name of the table (schemaname.childtablename) given the check constraint values.
|
122
124
|
#
|
123
125
|
def partition_table_name(*partition_key_values)
|
124
126
|
return configurator.table_name(*partition_key_values)
|
125
127
|
end
|
126
128
|
|
127
129
|
#
|
128
|
-
#
|
130
|
+
# Create a single child table.
|
129
131
|
#
|
130
132
|
def create_partition_table(*partition_key_values)
|
131
133
|
create_table(configurator.table_name(*partition_key_values), {
|
@@ -137,14 +139,14 @@ module Partitioned
|
|
137
139
|
end
|
138
140
|
|
139
141
|
#
|
140
|
-
#
|
142
|
+
# Remove a specific single child table.
|
141
143
|
#
|
142
144
|
def drop_partition_table(*partition_key_values)
|
143
145
|
drop_table(configurator.table_name(*partition_key_values))
|
144
146
|
end
|
145
147
|
|
146
148
|
#
|
147
|
-
#
|
149
|
+
# Add indexes that must exist on child tables. Only leaf child tables
|
148
150
|
# need indexes as parent table indexes are not used in postgres.
|
149
151
|
#
|
150
152
|
def add_partition_table_index(*partition_key_values)
|
@@ -159,31 +161,31 @@ module Partitioned
|
|
159
161
|
end
|
160
162
|
|
161
163
|
#
|
162
|
-
#
|
164
|
+
# Used when creating the name of a SQL rule.
|
163
165
|
#
|
164
166
|
def parent_table_rule_name(name, suffix = "rule", *partition_key_values)
|
165
167
|
return "#{configurator.parent_table_name(*partition_key_values).gsub(/[.]/, '_')}_#{name}_#{suffix}"
|
166
168
|
end
|
167
169
|
|
168
170
|
#
|
169
|
-
#
|
171
|
+
# Used to create index names.
|
170
172
|
#
|
171
173
|
def index_name(name, *partition_key_values)
|
172
174
|
return "#{configurator.part_name(*partition_key_values)}_#{name}_idx"
|
173
175
|
end
|
174
176
|
|
175
177
|
#
|
176
|
-
#
|
178
|
+
# Used to create index names.
|
177
179
|
#
|
178
180
|
def unique_index_name(name, *partition_key_values)
|
179
181
|
return "#{configurator.part_name(*partition_key_values)}_#{name}_udx"
|
180
182
|
end
|
181
183
|
|
182
184
|
#
|
183
|
-
#
|
185
|
+
# This is here for derived classes to set up references to added columns
|
184
186
|
# (or columns in the parent that need foreign key constraints).
|
185
187
|
#
|
186
|
-
#
|
188
|
+
# Foreign keys are not inherited in postgres. So, a parent table
|
187
189
|
# of the form:
|
188
190
|
#
|
189
191
|
# -- this is the referenced table
|
@@ -211,12 +213,12 @@ module Partitioned
|
|
211
213
|
# create table employees_of_company_2 ( CHECK ( company_id = 2 ) ) INHERITS (employees);
|
212
214
|
# create table employees_of_company_3 ( CHECK ( company_id = 3 ) ) INHERITS (employees);
|
213
215
|
#
|
214
|
-
#
|
216
|
+
# Since postgres does not inherit referential integrity from parent tables, the following
|
215
217
|
# insert will work:
|
216
218
|
# insert into employees_of_company_1 (name, company_id, supervisor_id) values ('joe', 1, 10);
|
217
219
|
# even if there is no record in companies with id = 1 and there is no record in employees with id = 10
|
218
220
|
#
|
219
|
-
#
|
221
|
+
# For proper referential integrity handling you must do the following:
|
220
222
|
# ALTER TABLE employees_of_company_1 add foreign key (company_id) references companies(id)
|
221
223
|
# ALTER TABLE employees_of_company_2 add foreign key (company_id) references companies(id)
|
222
224
|
# ALTER TABLE employees_of_company_3 add foreign key (company_id) references companies(id)
|
@@ -225,7 +227,7 @@ module Partitioned
|
|
225
227
|
# ALTER TABLE employees_of_company_2 add foreign key (supervisor_id) references employees_of_company_2(id)
|
226
228
|
# ALTER TABLE employees_of_company_3 add foreign key (supervisor_id) references employees_of_company_3(id)
|
227
229
|
#
|
228
|
-
#
|
230
|
+
# The second set of alter tables brings up a good another consideration about postgres references and partitions.
|
229
231
|
# postgres will not follow references to a child table. So, a foreign key reference to "employees" in this
|
230
232
|
# set of alter statements would not work because postgres would expect the table "employees" to have
|
231
233
|
# the specific referenced record, but the record really exists in a child of employees. So, the alter statement
|