partitioned 0.8.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README +85 -36
- data/Rakefile +3 -0
- data/examples/README +46 -18
- data/lib/monkey_patch_activerecord.rb +14 -8
- data/lib/monkey_patch_postgres.rb +46 -13
- data/lib/partitioned/active_record_overrides.rb +13 -5
- data/lib/partitioned/bulk_methods_mixin.rb +91 -146
- data/lib/partitioned/by_created_at.rb +3 -1
- data/lib/partitioned/by_foreign_key.rb +5 -0
- data/lib/partitioned/by_id.rb +10 -4
- data/lib/partitioned/by_integer_field.rb +9 -0
- data/lib/partitioned/by_monthly_time_field.rb +8 -1
- data/lib/partitioned/by_time_field.rb +16 -8
- data/lib/partitioned/by_weekly_time_field.rb +6 -3
- data/lib/partitioned/multi_level/configurator/data.rb +1 -0
- data/lib/partitioned/multi_level/configurator/dsl.rb +11 -0
- data/lib/partitioned/multi_level/configurator/reader.rb +18 -0
- data/lib/partitioned/multi_level/partition_manager.rb +13 -4
- data/lib/partitioned/multi_level.rb +3 -1
- data/lib/partitioned/partitioned_base/configurator/data.rb +10 -1
- data/lib/partitioned/partitioned_base/configurator/dsl.rb +20 -15
- data/lib/partitioned/partitioned_base/configurator/reader.rb +3 -0
- data/lib/partitioned/partitioned_base/configurator.rb +4 -0
- data/lib/partitioned/partitioned_base/partition_manager.rb +17 -15
- data/lib/partitioned/partitioned_base/sql_adapter.rb +25 -23
- data/lib/partitioned/partitioned_base.rb +112 -41
- data/lib/partitioned/version.rb +2 -1
- data/partitioned.gemspec +3 -2
- metadata +68 -73
@@ -4,22 +4,22 @@ module Partitioned
|
|
4
4
|
#
|
5
5
|
# The Domain Specific Language manager for configuring partitioning.
|
6
6
|
#
|
7
|
-
# example:
|
7
|
+
# @example of use:
|
8
8
|
#
|
9
|
-
#
|
10
|
-
#
|
11
|
-
#
|
12
|
-
#
|
9
|
+
# class Employee < Partitioned::ByCreatedAt
|
10
|
+
# partitioned do |partition|
|
11
|
+
# partition.index :id, :unique => true
|
12
|
+
# partition.foreign_key :company_id
|
13
|
+
# end
|
13
14
|
# end
|
14
|
-
# end
|
15
15
|
#
|
16
|
-
#
|
16
|
+
# in the above example, block:
|
17
17
|
#
|
18
|
-
#
|
19
|
-
#
|
20
|
-
#
|
18
|
+
# partitioned do |partition|
|
19
|
+
# ...
|
20
|
+
# end
|
21
21
|
#
|
22
|
-
#
|
22
|
+
# Scopes a set of "partition directives". The directives are accessed via the block parameter 'partition'
|
23
23
|
#
|
24
24
|
# Directives parameters have two forms, a canonical form which takes a set of parameters
|
25
25
|
# and a dynamic form which takes a single parameter which is either a string that should be interpolated or
|
@@ -36,10 +36,11 @@ module Partitioned
|
|
36
36
|
# end
|
37
37
|
#
|
38
38
|
# partitioned do |partition|
|
39
|
-
# partition.on lambda{|model| return model.partition_time_field}
|
39
|
+
# partition.on lambda{ |model| return model.partition_time_field}
|
40
40
|
# partition.constraint lambda{|model,time_field_value|
|
41
|
-
#
|
42
|
-
#
|
41
|
+
# return "#{model.partition_time_field} >= '#{time_field_value.strftime}' and
|
42
|
+
# #{model.partition_time_field} < '#{(time_field_value + 1.day).strftime}'"
|
43
|
+
# }
|
43
44
|
# end
|
44
45
|
# end
|
45
46
|
#
|
@@ -134,6 +135,9 @@ module Partitioned
|
|
134
135
|
# resolve to FavoriteEmployee.index_field and be :baz as expected.
|
135
136
|
#
|
136
137
|
class Dsl
|
138
|
+
#
|
139
|
+
# raised when a partitioned DSL directive's parameters are considered invalid
|
140
|
+
#
|
137
141
|
class InvalidConfiguratorDirectiveValue < StandardError
|
138
142
|
def initialize(model, table_name, directive, value, explanation)
|
139
143
|
super("#{model.name} [#{table_name}] invalid value '#{value}' for partitioned directive '#{directive}'. #{explanation}")
|
@@ -254,7 +258,8 @@ module Partitioned
|
|
254
258
|
if referencing_field.is_a? Proc
|
255
259
|
data.foreign_keys << referencing_field
|
256
260
|
else
|
257
|
-
data.foreign_keys << Partitioned::PartitionedBase::Configurator::Data::ForeignKey.
|
261
|
+
data.foreign_keys << Partitioned::PartitionedBase::Configurator::Data::ForeignKey.
|
262
|
+
new(referencing_field, referenced_table, referenced_field)
|
258
263
|
end
|
259
264
|
end
|
260
265
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
1
3
|
module Partitioned
|
2
4
|
class PartitionedBase
|
3
5
|
#
|
@@ -13,7 +15,7 @@ module Partitioned
|
|
13
15
|
end
|
14
16
|
|
15
17
|
#
|
16
|
-
#
|
18
|
+
# Drop partitions that are no longer necessary.
|
17
19
|
# uses #old_partition_key_values_set as the list of
|
18
20
|
# partitions to remove.
|
19
21
|
#
|
@@ -24,7 +26,7 @@ module Partitioned
|
|
24
26
|
end
|
25
27
|
|
26
28
|
#
|
27
|
-
#
|
29
|
+
# Create partitions that are needed (probably to handle data that
|
28
30
|
# will be inserted into the database within the next few weeks).
|
29
31
|
# uses #new_partition_key_value_set to determine the key values
|
30
32
|
# for the specific child tables to create.
|
@@ -36,7 +38,7 @@ module Partitioned
|
|
36
38
|
end
|
37
39
|
|
38
40
|
#
|
39
|
-
#
|
41
|
+
# Create any partition tables from a list. the partition tables must
|
40
42
|
# not already exist and its schema must already exist.
|
41
43
|
#
|
42
44
|
def create_new_partition_tables(enumerable)
|
@@ -46,7 +48,7 @@ module Partitioned
|
|
46
48
|
end
|
47
49
|
|
48
50
|
#
|
49
|
-
#
|
51
|
+
# The once called function to prepare a parent table for partitioning as well
|
50
52
|
# as create the schema that the child tables will be placed in.
|
51
53
|
#
|
52
54
|
def create_infrastructure
|
@@ -56,41 +58,41 @@ module Partitioned
|
|
56
58
|
|
57
59
|
protected
|
58
60
|
#
|
59
|
-
#
|
61
|
+
# An array of key values (each key value is an array of keys) that represent
|
60
62
|
# the child partitions that should be created.
|
61
63
|
#
|
62
|
-
#
|
63
|
-
# the database with new soon-to-be needed child tables
|
64
|
+
# Used by #create_new_partitions and generally called once a day to update
|
65
|
+
# the database with new soon-to-be needed child tables.
|
64
66
|
#
|
65
|
-
#
|
67
|
+
# Typically overridden by the concrete class as this is pure business logic.
|
66
68
|
#
|
67
69
|
def new_partition_key_values_set
|
68
70
|
[]
|
69
71
|
end
|
70
72
|
|
71
73
|
#
|
72
|
-
#
|
74
|
+
# An array of key values (each key value is an array of keys) that represent
|
73
75
|
# the child partitions that should be dropped because they are no longer needed.
|
74
76
|
#
|
75
|
-
#
|
76
|
-
# unneeded child tables
|
77
|
+
# Used by #drop_old_partitions and generally called once a day to clean up
|
78
|
+
# unneeded child tables.
|
77
79
|
#
|
78
|
-
#
|
80
|
+
# Typically overridden by the concrete class as this is pure business logic.
|
79
81
|
#
|
80
82
|
def old_partition_key_values_set
|
81
83
|
[]
|
82
84
|
end
|
83
85
|
|
84
86
|
#
|
85
|
-
#
|
86
|
-
# the key value(s) of its check constraint columns
|
87
|
+
# Remove a specific partition from the database given
|
88
|
+
# the key value(s) of its check constraint columns.
|
87
89
|
#
|
88
90
|
def drop_old_partition(*partition_key_values)
|
89
91
|
drop_partition_table(*partition_key_values)
|
90
92
|
end
|
91
93
|
|
92
94
|
#
|
93
|
-
#
|
95
|
+
# Create a specific child table that does not currently
|
94
96
|
# exist and whose schema (the schema that the table exists in)
|
95
97
|
# also already exists (#create_infrastructure is designed to
|
96
98
|
# create this).
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
1
3
|
module Partitioned
|
2
4
|
class PartitionedBase
|
3
5
|
#
|
@@ -12,7 +14,7 @@ module Partitioned
|
|
12
14
|
end
|
13
15
|
|
14
16
|
#
|
15
|
-
#
|
17
|
+
# Ensure our function for warning about improper partition usage is in place.
|
16
18
|
#
|
17
19
|
# Name: always_fail_on_insert(text); Type: FUNCTION; Schema: public
|
18
20
|
#
|
@@ -43,7 +45,7 @@ module Partitioned
|
|
43
45
|
end
|
44
46
|
|
45
47
|
#
|
46
|
-
#
|
48
|
+
# Does a specific child partition exist.
|
47
49
|
#
|
48
50
|
def partition_exists?(*partition_key_values)
|
49
51
|
return find(:first,
|
@@ -56,22 +58,22 @@ module Partitioned
|
|
56
58
|
end
|
57
59
|
|
58
60
|
#
|
59
|
-
#
|
61
|
+
# Returns an array of partition table names from last to first limited to
|
60
62
|
# the number of entries requested by its first parameter.
|
61
63
|
#
|
62
|
-
#
|
64
|
+
# The magic here is in the overridden method "last_n_partitions_order_by_clause"
|
63
65
|
# which is designed to order a list of partition table names (table names without
|
64
66
|
# their schema name) from last to first.
|
65
67
|
#
|
66
|
-
#
|
68
|
+
# If the child table names are the format "pYYYYMMDD" where YYYY is a four digit year, MM is
|
67
69
|
# a month number and DD is a day number, you would use the following to order from last to
|
68
70
|
# first:
|
69
71
|
# tablename desc
|
70
72
|
#
|
71
|
-
#
|
73
|
+
# For child table names of the format "pXXXX" where XXXX is a number, you may want something like:
|
72
74
|
# substring(tablename, 2)::integer desc
|
73
75
|
#
|
74
|
-
#
|
76
|
+
# For clarity, the sql executed is:
|
75
77
|
# select tablename from pg_tables where schemaname = $1 order by $2 limit $3
|
76
78
|
# where:
|
77
79
|
# $1 = the name of schema (foos_partitions)
|
@@ -88,18 +90,18 @@ module Partitioned
|
|
88
90
|
end
|
89
91
|
|
90
92
|
#
|
91
|
-
#
|
93
|
+
# Override this or order the tables from last (greatest value? greatest date?) to first.
|
92
94
|
#
|
93
95
|
def last_n_partitions_order_by_clause
|
94
96
|
return configurator.last_n_partitions_order_by_clause
|
95
97
|
end
|
96
98
|
|
97
99
|
#
|
98
|
-
#
|
100
|
+
# Used to create the parent table rule to ensure.
|
99
101
|
#
|
100
|
-
#
|
102
|
+
# This will cause an error on attempt to insert into the parent table.
|
101
103
|
#
|
102
|
-
#
|
104
|
+
# We want all records to exist in one of the child tables so the
|
103
105
|
# query planner can optimize access to the records.
|
104
106
|
#
|
105
107
|
def add_parent_table_rules(*partition_key_values)
|
@@ -118,14 +120,14 @@ module Partitioned
|
|
118
120
|
end
|
119
121
|
|
120
122
|
#
|
121
|
-
#
|
123
|
+
# The name of the table (schemaname.childtablename) given the check constraint values.
|
122
124
|
#
|
123
125
|
def partition_table_name(*partition_key_values)
|
124
126
|
return configurator.table_name(*partition_key_values)
|
125
127
|
end
|
126
128
|
|
127
129
|
#
|
128
|
-
#
|
130
|
+
# Create a single child table.
|
129
131
|
#
|
130
132
|
def create_partition_table(*partition_key_values)
|
131
133
|
create_table(configurator.table_name(*partition_key_values), {
|
@@ -137,14 +139,14 @@ module Partitioned
|
|
137
139
|
end
|
138
140
|
|
139
141
|
#
|
140
|
-
#
|
142
|
+
# Remove a specific single child table.
|
141
143
|
#
|
142
144
|
def drop_partition_table(*partition_key_values)
|
143
145
|
drop_table(configurator.table_name(*partition_key_values))
|
144
146
|
end
|
145
147
|
|
146
148
|
#
|
147
|
-
#
|
149
|
+
# Add indexes that must exist on child tables. Only leaf child tables
|
148
150
|
# need indexes as parent table indexes are not used in postgres.
|
149
151
|
#
|
150
152
|
def add_partition_table_index(*partition_key_values)
|
@@ -159,31 +161,31 @@ module Partitioned
|
|
159
161
|
end
|
160
162
|
|
161
163
|
#
|
162
|
-
#
|
164
|
+
# Used when creating the name of a SQL rule.
|
163
165
|
#
|
164
166
|
def parent_table_rule_name(name, suffix = "rule", *partition_key_values)
|
165
167
|
return "#{configurator.parent_table_name(*partition_key_values).gsub(/[.]/, '_')}_#{name}_#{suffix}"
|
166
168
|
end
|
167
169
|
|
168
170
|
#
|
169
|
-
#
|
171
|
+
# Used to create index names.
|
170
172
|
#
|
171
173
|
def index_name(name, *partition_key_values)
|
172
174
|
return "#{configurator.part_name(*partition_key_values)}_#{name}_idx"
|
173
175
|
end
|
174
176
|
|
175
177
|
#
|
176
|
-
#
|
178
|
+
# Used to create index names.
|
177
179
|
#
|
178
180
|
def unique_index_name(name, *partition_key_values)
|
179
181
|
return "#{configurator.part_name(*partition_key_values)}_#{name}_udx"
|
180
182
|
end
|
181
183
|
|
182
184
|
#
|
183
|
-
#
|
185
|
+
# This is here for derived classes to set up references to added columns
|
184
186
|
# (or columns in the parent that need foreign key constraints).
|
185
187
|
#
|
186
|
-
#
|
188
|
+
# Foreign keys are not inherited in postgres. So, a parent table
|
187
189
|
# of the form:
|
188
190
|
#
|
189
191
|
# -- this is the referenced table
|
@@ -211,12 +213,12 @@ module Partitioned
|
|
211
213
|
# create table employees_of_company_2 ( CHECK ( company_id = 2 ) ) INHERITS (employees);
|
212
214
|
# create table employees_of_company_3 ( CHECK ( company_id = 3 ) ) INHERITS (employees);
|
213
215
|
#
|
214
|
-
#
|
216
|
+
# Since postgres does not inherit referential integrity from parent tables, the following
|
215
217
|
# insert will work:
|
216
218
|
# insert into employees_of_company_1 (name, company_id, supervisor_id) values ('joe', 1, 10);
|
217
219
|
# even if there is no record in companies with id = 1 and there is no record in employees with id = 10
|
218
220
|
#
|
219
|
-
#
|
221
|
+
# For proper referential integrity handling you must do the following:
|
220
222
|
# ALTER TABLE employees_of_company_1 add foreign key (company_id) references companies(id)
|
221
223
|
# ALTER TABLE employees_of_company_2 add foreign key (company_id) references companies(id)
|
222
224
|
# ALTER TABLE employees_of_company_3 add foreign key (company_id) references companies(id)
|
@@ -225,7 +227,7 @@ module Partitioned
|
|
225
227
|
# ALTER TABLE employees_of_company_2 add foreign key (supervisor_id) references employees_of_company_2(id)
|
226
228
|
# ALTER TABLE employees_of_company_3 add foreign key (supervisor_id) references employees_of_company_3(id)
|
227
229
|
#
|
228
|
-
#
|
230
|
+
# The second set of alter tables brings up a good another consideration about postgres references and partitions.
|
229
231
|
# postgres will not follow references to a child table. So, a foreign key reference to "employees" in this
|
230
232
|
# set of alter statements would not work because postgres would expect the table "employees" to have
|
231
233
|
# the specific referenced record, but the record really exists in a child of employees. So, the alter statement
|