partitioned 1.1.3 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile.lock +6 -6
- data/LICENSE +1 -1
- data/lib/monkey_patch_activerecord.rb +12 -7
- data/lib/monkey_patch_postgres.rb +3 -0
- data/lib/monkey_patch_redshift.rb +114 -0
- data/lib/partitioned/multi_level/configurator/reader.rb +39 -21
- data/lib/partitioned/multi_level/partition_manager.rb +2 -2
- data/lib/partitioned/partitioned_base/configurator/data.rb +4 -1
- data/lib/partitioned/partitioned_base/configurator/dsl.rb +10 -0
- data/lib/partitioned/partitioned_base/configurator/reader.rb +10 -1
- data/lib/partitioned/partitioned_base/partition_manager.rb +1 -0
- data/lib/partitioned/partitioned_base/redshift_sql_adapter.rb +269 -0
- data/lib/partitioned/partitioned_base/sql_adapter.rb +6 -5
- data/lib/partitioned/partitioned_base.rb +23 -19
- data/lib/partitioned/version.rb +1 -1
- data/lib/partitioned.rb +2 -0
- data/partitioned.gemspec +1 -1
- metadata +5 -3
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
partitioned (1.1.
|
4
|
+
partitioned (1.1.4)
|
5
5
|
bulk_data_methods (= 1.0.0)
|
6
6
|
pg
|
7
7
|
rails (>= 3.2.8)
|
@@ -48,7 +48,7 @@ GEM
|
|
48
48
|
hike (1.2.1)
|
49
49
|
i18n (0.6.1)
|
50
50
|
journey (1.0.4)
|
51
|
-
jquery-rails (2.
|
51
|
+
jquery-rails (2.2.0)
|
52
52
|
railties (>= 3.0, < 5.0)
|
53
53
|
thor (>= 0.14, < 2.0)
|
54
54
|
json (1.7.6)
|
@@ -56,14 +56,14 @@ GEM
|
|
56
56
|
i18n (>= 0.4.0)
|
57
57
|
mime-types (~> 1.16)
|
58
58
|
treetop (~> 1.4.8)
|
59
|
-
mime-types (1.
|
59
|
+
mime-types (1.20.1)
|
60
60
|
multi_json (1.5.0)
|
61
61
|
pg (0.14.1)
|
62
62
|
polyglot (0.3.3)
|
63
63
|
rack (1.4.4)
|
64
64
|
rack-cache (1.2)
|
65
65
|
rack (>= 0.4)
|
66
|
-
rack-ssl (1.3.
|
66
|
+
rack-ssl (1.3.3)
|
67
67
|
rack
|
68
68
|
rack-test (0.6.2)
|
69
69
|
rack (>= 1.0)
|
@@ -88,7 +88,7 @@ GEM
|
|
88
88
|
rspec-core (2.12.2)
|
89
89
|
rspec-expectations (2.12.1)
|
90
90
|
diff-lcs (~> 1.1.3)
|
91
|
-
rspec-mocks (2.12.
|
91
|
+
rspec-mocks (2.12.2)
|
92
92
|
rspec-rails (2.12.2)
|
93
93
|
actionpack (>= 3.0)
|
94
94
|
activesupport (>= 3.0)
|
@@ -101,7 +101,7 @@ GEM
|
|
101
101
|
multi_json (~> 1.0)
|
102
102
|
rack (~> 1.0)
|
103
103
|
tilt (~> 1.1, != 1.3.0)
|
104
|
-
thor (0.
|
104
|
+
thor (0.17.0)
|
105
105
|
tilt (1.3.3)
|
106
106
|
treetop (1.4.12)
|
107
107
|
polyglot
|
data/LICENSE
CHANGED
@@ -17,21 +17,26 @@ module ActiveRecord
|
|
17
17
|
# that no changes should be made (since they can't be persisted).
|
18
18
|
def destroy
|
19
19
|
destroy_associations
|
20
|
-
|
20
|
+
|
21
21
|
if persisted?
|
22
22
|
IdentityMap.remove(self) if IdentityMap.enabled?
|
23
23
|
pk = self.class.primary_key
|
24
24
|
column = self.class.columns_hash[pk]
|
25
25
|
substitute = connection.substitute_at(column, 0)
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
26
|
+
|
27
|
+
if self.class.respond_to?(:dynamic_arel_table)
|
28
|
+
using_arel_table = dynamic_arel_table()
|
29
|
+
relation = ActiveRecord::Relation.new(self.class, using_arel_table).
|
30
|
+
where(using_arel_table[pk].eq(substitute))
|
31
|
+
else
|
32
|
+
using_arel_table = self.class.arel_table
|
33
|
+
relation = self.class.unscoped.where(using_arel_table[pk].eq(substitute))
|
34
|
+
end
|
35
|
+
|
31
36
|
relation.bind_values = [[column, id]]
|
32
37
|
relation.delete_all
|
33
38
|
end
|
34
|
-
|
39
|
+
|
35
40
|
@destroyed = true
|
36
41
|
freeze
|
37
42
|
end
|
@@ -29,6 +29,9 @@ module ActiveRecord::ConnectionAdapters
|
|
29
29
|
# to take advantage of these SQL builders.
|
30
30
|
#
|
31
31
|
class PostgreSQLAdapter < AbstractAdapter
|
32
|
+
def partitioned_sql_adapter(model)
|
33
|
+
return Partitioned::PartitionedBase::SqlAdapter.new(model)
|
34
|
+
end
|
32
35
|
|
33
36
|
#
|
34
37
|
# Returns the sequence name for a table's primary key or some other specified key.
|
@@ -0,0 +1,114 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'active_record/base'
|
3
|
+
require 'active_record/connection_adapters/abstract_adapter'
|
4
|
+
require 'active_record/connection_adapters/redshift_adapter'
|
5
|
+
|
6
|
+
#
|
7
|
+
# Patching {ActiveRecord::ConnectionAdapters::TableDefinition} and
|
8
|
+
# {ActiveRecord::ConnectionAdapters::PostgreSQLAdapter} to add functionality
|
9
|
+
# needed to abstract partition specific SQL statements.
|
10
|
+
#
|
11
|
+
module ActiveRecord::ConnectionAdapters
|
12
|
+
#
|
13
|
+
# Patches extending the postgres adapter with new operations for managing
|
14
|
+
# sequences (and sets of sequence values), schemas and foreign keys.
|
15
|
+
# These should go into AbstractAdapter allowing any database adapter
|
16
|
+
# to take advantage of these SQL builders.
|
17
|
+
#
|
18
|
+
class RedshiftAdapter < AbstractAdapter
|
19
|
+
def partitioned_sql_adapter(model)
|
20
|
+
return Partitioned::PartitionedBase::RedshiftSqlAdapter.new(model)
|
21
|
+
end
|
22
|
+
|
23
|
+
#
|
24
|
+
# Returns the sequence name for a table's primary key or some other specified key.
|
25
|
+
#
|
26
|
+
# the default version strips off the schema name on the table (if it exists), as:
|
27
|
+
# serial_sequence(table_name, pk || 'id').split('.').last
|
28
|
+
# i can't see any good reason for that -- in fact, it seems completely
|
29
|
+
# broken -- if you have a table public.foos and other.foos, you'll fail to
|
30
|
+
# get the correct schema if you fetch the default schema name from model
|
31
|
+
# associated with other.foos
|
32
|
+
#
|
33
|
+
def default_sequence_name(table_name, pk = nil) #:nodoc:
|
34
|
+
serial_sequence(table_name, pk || 'id')
|
35
|
+
rescue ActiveRecord::StatementInvalid => e
|
36
|
+
"#{table_name}_#{pk || 'id'}_seq"
|
37
|
+
end
|
38
|
+
|
39
|
+
#
|
40
|
+
# Get the next value in a sequence. Used on INSERT operation for
|
41
|
+
# partitioning like by_id because the ID is required before the insert
|
42
|
+
# so that the specific child table is known ahead of time.
|
43
|
+
#
|
44
|
+
# @param [String] sequence_name the name of the sequence to fetch the next value from
|
45
|
+
# @return [Integer] the value from the sequence
|
46
|
+
def next_sequence_value(sequence_name)
|
47
|
+
return execute("select nextval('#{sequence_name}')").field_values("nextval").first.to_i
|
48
|
+
end
|
49
|
+
|
50
|
+
#
|
51
|
+
# Get the some next values in a sequence.
|
52
|
+
#
|
53
|
+
# @param [String] sequence_name the name of the sequence to fetch the next values from
|
54
|
+
# @param [Integer] batch_size count of values.
|
55
|
+
# @return [Array<Integer>] an array of values from the sequence
|
56
|
+
def next_sequence_values(sequence_name, batch_size)
|
57
|
+
result = execute("select nextval('#{sequence_name}') from generate_series(1, #{batch_size})")
|
58
|
+
return result.field_values("nextval").map(&:to_i)
|
59
|
+
end
|
60
|
+
|
61
|
+
#
|
62
|
+
# Causes active resource to fetch the primary key for the table (using next_sequence_value())
|
63
|
+
# just before an insert. We need the prefetch to happen but we don't have enough information
|
64
|
+
# here to determine if it should happen, so Relation::insert has been modified to request of
|
65
|
+
# the ActiveRecord::Base derived class if it requires a prefetch.
|
66
|
+
#
|
67
|
+
# @param [String] table_name the table name to query
|
68
|
+
# @return [Boolean] returns true if the table should have its primary key prefetched.
|
69
|
+
def prefetch_primary_key?(table_name)
|
70
|
+
return false
|
71
|
+
end
|
72
|
+
|
73
|
+
#
|
74
|
+
# Creates a schema given a name.
|
75
|
+
#
|
76
|
+
# @param [String] name the name of the schema.
|
77
|
+
# @param [Hash] options ({}) options for creating a schema
|
78
|
+
# @option options [Boolean] :unless_exists (false) check if schema exists.
|
79
|
+
# @return [optional] undefined
|
80
|
+
def create_schema(name, options = {})
|
81
|
+
if options[:unless_exists]
|
82
|
+
return if execute("select count(*) from pg_namespace where nspname = '#{name}'").getvalue(0,0).to_i > 0
|
83
|
+
end
|
84
|
+
execute("CREATE SCHEMA #{name}")
|
85
|
+
end
|
86
|
+
|
87
|
+
#
|
88
|
+
# Drop a schema given a name.
|
89
|
+
#
|
90
|
+
# @param [String] name the name of the schema.
|
91
|
+
# @param [Hash] options ({}) options for dropping a schema
|
92
|
+
# @option options [Boolean] :if_exists (false) check if schema exists.
|
93
|
+
# @option options [Boolean] :cascade (false) drop dependant objects
|
94
|
+
# @return [optional] undefined
|
95
|
+
def drop_schema(name, options = {})
|
96
|
+
if options[:if_exists]
|
97
|
+
return if execute("select count(*) from pg_namespace where nspname = '#{name}'").getvalue(0,0).to_i == 0
|
98
|
+
end
|
99
|
+
execute("DROP SCHEMA #{name}#{' cascade' if options[:cascade]}")
|
100
|
+
end
|
101
|
+
|
102
|
+
#
|
103
|
+
# Add foreign key constraint to table.
|
104
|
+
#
|
105
|
+
# @param [String] referencing_table_name the name of the table containing the foreign key
|
106
|
+
# @param [String] referencing_field_name the name of foreign key column
|
107
|
+
# @param [String] referenced_table_name the name of the table referenced by the foreign key
|
108
|
+
# @param [String] referenced_field_name (:id) the name of the column referenced by the foreign key
|
109
|
+
# @return [optional] undefined
|
110
|
+
def add_foreign_key(referencing_table_name, referencing_field_name, referenced_table_name, referenced_field_name = :id)
|
111
|
+
execute("ALTER TABLE #{referencing_table_name} add foreign key (#{referencing_field_name}) references #{referenced_table_name}(#{referenced_field_name})")
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -5,6 +5,10 @@ module Partitioned
|
|
5
5
|
# {PartitionManager} to request partitioning information froma
|
6
6
|
# centralized source from multi level partitioned models
|
7
7
|
class Reader < Partitioned::PartitionedBase::Configurator::Reader
|
8
|
+
|
9
|
+
alias :base_collect_from_collection :collect_from_collection
|
10
|
+
alias :base_collect :collect
|
11
|
+
|
8
12
|
# configurator for a specific class level
|
9
13
|
UsingConfigurator = Struct.new(:model, :sliced_class, :dsl)
|
10
14
|
|
@@ -20,7 +24,7 @@ module Partitioned
|
|
20
24
|
# @return [Array<Symbol>] fields used to partition this model
|
21
25
|
def on_fields
|
22
26
|
unless @on_fields
|
23
|
-
@on_fields =
|
27
|
+
@on_fields = collect(&:on_field).map(&:to_sym)
|
24
28
|
end
|
25
29
|
return @on_fields
|
26
30
|
end
|
@@ -67,6 +71,35 @@ module Partitioned
|
|
67
71
|
return using_configurator(index).check_constraint(value)
|
68
72
|
end
|
69
73
|
|
74
|
+
def indexes(*partition_key_values)
|
75
|
+
bag = {}
|
76
|
+
partition_key_values.each_with_index do |key_value, index|
|
77
|
+
bag.merge!(using_configurator(index).indexes(key_value))
|
78
|
+
end
|
79
|
+
base_collect_from_collection(*partition_key_values, &:indexes).inject(bag) do |bag, data_index|
|
80
|
+
bag[data_index.field] = (data_index.options || {}) unless data_index.blank?
|
81
|
+
bag
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
#
|
86
|
+
# Foreign keys to create on each leaf partition.
|
87
|
+
#
|
88
|
+
def foreign_keys(*partition_key_values)
|
89
|
+
set = Set.new
|
90
|
+
partition_key_values.each_with_index do |key_value, index|
|
91
|
+
set.merge(using_configurator(index).foreign_keys(key_value))
|
92
|
+
end
|
93
|
+
base_collect_from_collection(*partition_key_values, &:foreign_keys).inject(set) do |set, new_items|
|
94
|
+
if new_items.is_a? Array
|
95
|
+
set += new_items
|
96
|
+
else
|
97
|
+
set += [new_items]
|
98
|
+
end
|
99
|
+
set
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
70
103
|
#
|
71
104
|
# The name of the child table without the schema name or name prefix.
|
72
105
|
#
|
@@ -117,14 +150,14 @@ module Partitioned
|
|
117
150
|
|
118
151
|
def using_classes
|
119
152
|
unless @using_classes
|
120
|
-
@using_classes =
|
153
|
+
@using_classes = base_collect_from_collection(&:using_classes).inject([]) do |array,new_items|
|
121
154
|
array += [*new_items]
|
122
155
|
end.to_a
|
123
156
|
end
|
124
157
|
return @using_classes
|
125
158
|
end
|
126
159
|
|
127
|
-
def
|
160
|
+
def collect(*partition_key_values, &block)
|
128
161
|
values = []
|
129
162
|
using_configurators.each do |using_configurator|
|
130
163
|
data = using_configurator.dsl.data
|
@@ -137,25 +170,10 @@ module Partitioned
|
|
137
170
|
values << intermediate_value unless intermediate_value.blank?
|
138
171
|
end
|
139
172
|
end
|
140
|
-
return values
|
141
|
-
end
|
142
|
-
|
143
|
-
def using_collect_first(*partition_key_values, &block)
|
144
|
-
using_configurators.each do |using_configurator|
|
145
|
-
data = using_configurator.dsl.data
|
146
|
-
intermediate_value = block.call(data) rescue nil
|
147
|
-
if intermediate_value.is_a? Proc
|
148
|
-
return intermediate_value.call(using_configurator.model, *partition_key_values)
|
149
|
-
elsif intermediate_value.is_a? String
|
150
|
-
return eval("\"#{intermediate_value}\"")
|
151
|
-
else
|
152
|
-
return intermediate_value unless intermediate_value.nil?
|
153
|
-
end
|
154
|
-
end
|
155
|
-
return nil
|
173
|
+
return base_collect(*partition_key_values, &block) + values
|
156
174
|
end
|
157
175
|
|
158
|
-
def
|
176
|
+
def collect_from_collection(*partition_key_values, &block)
|
159
177
|
values = []
|
160
178
|
using_configurators.each do |using_configurator|
|
161
179
|
data = using_configurator.dsl.data
|
@@ -171,7 +189,7 @@ module Partitioned
|
|
171
189
|
end
|
172
190
|
end
|
173
191
|
end
|
174
|
-
return values
|
192
|
+
return base_collect_from_collection(*partition_key_values, &block) + values
|
175
193
|
end
|
176
194
|
|
177
195
|
end
|
@@ -17,8 +17,6 @@ module Partitioned
|
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
|
-
protected
|
21
|
-
|
22
20
|
#
|
23
21
|
# Create a specific child table that does not currently
|
24
22
|
# exist and whose schema (the schema that the table exists in)
|
@@ -37,6 +35,8 @@ module Partitioned
|
|
37
35
|
end
|
38
36
|
end
|
39
37
|
|
38
|
+
protected
|
39
|
+
|
40
40
|
#
|
41
41
|
# Is the table a child table without itself having any children.
|
42
42
|
# generally leaf tables are where all indexes and foreign key
|
@@ -42,7 +42,8 @@ module Partitioned
|
|
42
42
|
:schema_name, :name_prefix, :base_name,
|
43
43
|
:part_name, :table_name, :table_alias_name, :parent_table_schema_name,
|
44
44
|
:parent_table_name, :check_constraint, :encoded_name,
|
45
|
-
:janitorial_creates_needed, :janitorial_archives_needed, :janitorial_drops_needed
|
45
|
+
:janitorial_creates_needed, :janitorial_archives_needed, :janitorial_drops_needed,
|
46
|
+
:after_partition_table_create_hooks
|
46
47
|
|
47
48
|
def initialize
|
48
49
|
@on_field = nil
|
@@ -70,6 +71,8 @@ module Partitioned
|
|
70
71
|
@janitorial_creates_needed = nil
|
71
72
|
@janitorial_archives_needed = nil
|
72
73
|
@janitorial_drops_needed = nil
|
74
|
+
|
75
|
+
@after_partition_table_create_hooks = []
|
73
76
|
end
|
74
77
|
end
|
75
78
|
end
|
@@ -263,6 +263,16 @@ module Partitioned
|
|
263
263
|
end
|
264
264
|
end
|
265
265
|
|
266
|
+
def after_partition_table_create_hook(method_name)
|
267
|
+
if method_name.is_a? Proc
|
268
|
+
data.after_partition_table_create_hooks << method_name
|
269
|
+
else
|
270
|
+
# XXX should be a symbol
|
271
|
+
data.after_partition_table_create_hooks << lambda {|model,*partition_key_values| model.send(method_name, *partition_key_values)}
|
272
|
+
end
|
273
|
+
end
|
274
|
+
|
275
|
+
|
266
276
|
#
|
267
277
|
# Define the check constraint for a given child table.
|
268
278
|
#
|
@@ -30,6 +30,7 @@ module Partitioned
|
|
30
30
|
@janitorial_creates_needed = nil
|
31
31
|
@janitorial_archives_needed = nil
|
32
32
|
@janitorial_drops_needed = nil
|
33
|
+
@after_partition_table_create_hooks = nil
|
33
34
|
end
|
34
35
|
|
35
36
|
#
|
@@ -101,7 +102,11 @@ module Partitioned
|
|
101
102
|
# The full name of a child table defined by the partition key values.
|
102
103
|
#
|
103
104
|
def table_name(*partition_key_values)
|
104
|
-
|
105
|
+
if partition_key_values.length < 1
|
106
|
+
return model.table_name
|
107
|
+
else
|
108
|
+
return collect_first(*partition_key_values, &:table_name)
|
109
|
+
end
|
105
110
|
end
|
106
111
|
|
107
112
|
#
|
@@ -167,6 +172,10 @@ module Partitioned
|
|
167
172
|
return @janitorial_drops_needed
|
168
173
|
end
|
169
174
|
|
175
|
+
def run_after_partition_table_create_hooks(*partition_key_values)
|
176
|
+
collect_from_collection(*partition_key_values, &:after_partition_table_create_hooks)
|
177
|
+
end
|
178
|
+
|
170
179
|
protected
|
171
180
|
|
172
181
|
def configurators
|
@@ -127,6 +127,7 @@ module Partitioned
|
|
127
127
|
create_partition_table(*partition_key_values)
|
128
128
|
add_partition_table_index(*partition_key_values)
|
129
129
|
add_references_to_partition_table(*partition_key_values)
|
130
|
+
configurator.run_after_partition_table_create_hooks(*partition_key_values)
|
130
131
|
end
|
131
132
|
|
132
133
|
##
|
@@ -0,0 +1,269 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
3
|
+
module Partitioned
|
4
|
+
class PartitionedBase
|
5
|
+
#
|
6
|
+
# SqlAdapter
|
7
|
+
# manages requests of partitioned tables.
|
8
|
+
#
|
9
|
+
class RedshiftSqlAdapter
|
10
|
+
attr_reader :parent_table_class
|
11
|
+
|
12
|
+
def initialize(parent_table_class)
|
13
|
+
@parent_table_class = parent_table_class
|
14
|
+
end
|
15
|
+
|
16
|
+
#
|
17
|
+
# Ensure our function for warning about improper partition usage is in place.
|
18
|
+
#
|
19
|
+
# Name: always_fail_on_insert(text); Type: FUNCTION; Schema: public
|
20
|
+
#
|
21
|
+
# Used to raise an exception explaining why a specific insert (into a parent
|
22
|
+
# table which should never have records) should never be attempted.
|
23
|
+
#
|
24
|
+
def ensure_always_fail_on_insert_exists
|
25
|
+
# XXX nothing can be done here
|
26
|
+
end
|
27
|
+
|
28
|
+
#
|
29
|
+
# Child tables whose parent table is 'foos', typically exist in a schema named foos_partitions.
|
30
|
+
#
|
31
|
+
# *partition_key_values are needed here to support the use of multiple schemas to keep tables in.
|
32
|
+
#
|
33
|
+
def create_partition_schema(*partition_key_values)
|
34
|
+
create_schema(configurator.schema_name, :unless_exists => true)
|
35
|
+
end
|
36
|
+
|
37
|
+
#
|
38
|
+
# Does a specific child partition exist.
|
39
|
+
#
|
40
|
+
def partition_exists?(*partition_key_values)
|
41
|
+
return find(:first,
|
42
|
+
:from => "pg_tables",
|
43
|
+
:select => "count(*) as count",
|
44
|
+
:conditions => ["schemaname = ? and tablename = ?",
|
45
|
+
configurator.schema_name,
|
46
|
+
configurator.part_name(*partition_key_values)
|
47
|
+
]).count.to_i == 1
|
48
|
+
end
|
49
|
+
|
50
|
+
#
|
51
|
+
# Returns an array of partition table names from last to first limited to
|
52
|
+
# the number of entries requested by its first parameter.
|
53
|
+
#
|
54
|
+
# The magic here is in the overridden method "last_n_partitions_order_by_clause"
|
55
|
+
# which is designed to order a list of partition table names (table names without
|
56
|
+
# their schema name) from last to first.
|
57
|
+
#
|
58
|
+
# If the child table names are the format "pYYYYMMDD" where YYYY is a four digit year, MM is
|
59
|
+
# a month number and DD is a day number, you would use the following to order from last to
|
60
|
+
# first:
|
61
|
+
# tablename desc
|
62
|
+
#
|
63
|
+
# For child table names of the format "pXXXX" where XXXX is a number, you may want something like:
|
64
|
+
# substring(tablename, 2)::integer desc
|
65
|
+
#
|
66
|
+
# For clarity, the sql executed is:
|
67
|
+
# select tablename from pg_tables where schemaname = $1 order by $2 limit $3
|
68
|
+
# where:
|
69
|
+
# $1 = the name of schema (foos_partitions)
|
70
|
+
# $2 = the order by clause that would make the greatest table name listed first
|
71
|
+
# $3 = the parameter 'how_many'
|
72
|
+
#
|
73
|
+
def last_n_partition_names(how_many = 1)
|
74
|
+
return find(:all,
|
75
|
+
:from => "pg_tables",
|
76
|
+
:select => :tablename,
|
77
|
+
:conditions => ["schemaname = ?", configurator.schema_name],
|
78
|
+
:order => last_n_partitions_order_by_clause,
|
79
|
+
:limit => how_many).map(&:tablename)
|
80
|
+
end
|
81
|
+
|
82
|
+
#
|
83
|
+
# Override this or order the tables from last (greatest value? greatest date?) to first.
|
84
|
+
#
|
85
|
+
def last_n_partitions_order_by_clause
|
86
|
+
return configurator.last_partitions_order_by_clause
|
87
|
+
end
|
88
|
+
|
89
|
+
#
|
90
|
+
# Used to create the parent table rule to ensure.
|
91
|
+
#
|
92
|
+
# This will cause an error on attempt to insert into the parent table.
|
93
|
+
#
|
94
|
+
# We want all records to exist in one of the child tables so the
|
95
|
+
# query planner can optimize access to the records.
|
96
|
+
#
|
97
|
+
def add_parent_table_rules(*partition_key_values)
|
98
|
+
# XXX nothing can be done here
|
99
|
+
end
|
100
|
+
|
101
|
+
#
|
102
|
+
# The name of the table (schemaname.childtablename) given the check constraint values.
|
103
|
+
#
|
104
|
+
def partition_table_name(*partition_key_values)
|
105
|
+
return configurator.table_name(*partition_key_values)
|
106
|
+
end
|
107
|
+
|
108
|
+
#
|
109
|
+
# A reasonable alias for the partition table
|
110
|
+
#
|
111
|
+
def partition_table_alias_name(*partition_key_values)
|
112
|
+
return configurator.table_alias_name(*partition_key_values)
|
113
|
+
end
|
114
|
+
|
115
|
+
#
|
116
|
+
# Create a single child table.
|
117
|
+
#
|
118
|
+
def create_partition_table(*partition_key_values)
|
119
|
+
execute("select * into #{configurator.table_name(*partition_key_values)} from #{configurator.parent_table_name(*partition_key_values)} where false")
|
120
|
+
end
|
121
|
+
|
122
|
+
#
|
123
|
+
# Remove a specific single child table.
|
124
|
+
#
|
125
|
+
def drop_partition_table(*partition_key_values)
|
126
|
+
drop_table(configurator.table_name(*partition_key_values))
|
127
|
+
end
|
128
|
+
|
129
|
+
#
|
130
|
+
# Add indexes that must exist on child tables. Only leaf child tables
|
131
|
+
# need indexes as parent table indexes are not used in postgres.
|
132
|
+
#
|
133
|
+
def add_partition_table_index(*partition_key_values)
|
134
|
+
configurator.indexes(*partition_key_values).each do |field,options|
|
135
|
+
used_options = options.clone
|
136
|
+
unless used_options.has_key?(:name)
|
137
|
+
name = [*field].join('_')
|
138
|
+
used_options[:name] = used_options[:unique] ? unique_index_name(name, *partition_key_values) : index_name(name, *partition_key_values)
|
139
|
+
end
|
140
|
+
add_index(partition_table_name(*partition_key_values), field, used_options)
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
#
|
145
|
+
# Used when creating the name of a SQL rule.
|
146
|
+
#
|
147
|
+
def parent_table_rule_name(name, suffix = "rule", *partition_key_values)
|
148
|
+
return "#{configurator.table_name(*partition_key_values).gsub(/[.]/, '_')}_#{name}_#{suffix}"
|
149
|
+
end
|
150
|
+
|
151
|
+
#
|
152
|
+
# Used to create index names.
|
153
|
+
#
|
154
|
+
def index_name(name, *partition_key_values)
|
155
|
+
return "#{configurator.part_name(*partition_key_values)}_#{name}_idx"
|
156
|
+
end
|
157
|
+
|
158
|
+
#
|
159
|
+
# Used to create index names.
|
160
|
+
#
|
161
|
+
def unique_index_name(name, *partition_key_values)
|
162
|
+
return "#{configurator.part_name(*partition_key_values)}_#{name}_udx"
|
163
|
+
end
|
164
|
+
|
165
|
+
#
|
166
|
+
# This is here for derived classes to set up references to added columns
|
167
|
+
# (or columns in the parent that need foreign key constraints).
|
168
|
+
#
|
169
|
+
# Foreign keys are not inherited in postgres. So, a parent table
|
170
|
+
# of the form:
|
171
|
+
#
|
172
|
+
# -- this is the referenced table
|
173
|
+
# create table companies
|
174
|
+
# (
|
175
|
+
# id serial not null primary key,
|
176
|
+
# created_at timestamp not null default now(),
|
177
|
+
# updated_at timestamp,
|
178
|
+
# name text not null
|
179
|
+
# );
|
180
|
+
#
|
181
|
+
# -- this is the parent table
|
182
|
+
# create table employees
|
183
|
+
# (
|
184
|
+
# id serial not null primary key,
|
185
|
+
# created_at timestamp not null default now(),
|
186
|
+
# updated_at timestamp,
|
187
|
+
# name text not null,
|
188
|
+
# company_id integer not null references companies,
|
189
|
+
# supervisor_id integer not null references employees
|
190
|
+
# );
|
191
|
+
#
|
192
|
+
# -- some children
|
193
|
+
# create table employees_of_company_1 ( CHECK ( company_id = 1 ) ) INHERITS (employees);
|
194
|
+
# create table employees_of_company_2 ( CHECK ( company_id = 2 ) ) INHERITS (employees);
|
195
|
+
# create table employees_of_company_3 ( CHECK ( company_id = 3 ) ) INHERITS (employees);
|
196
|
+
#
|
197
|
+
# Since postgres does not inherit referential integrity from parent tables, the following
|
198
|
+
# insert will work:
|
199
|
+
# insert into employees_of_company_1 (name, company_id, supervisor_id) values ('joe', 1, 10);
|
200
|
+
# even if there is no record in companies with id = 1 and there is no record in employees with id = 10
|
201
|
+
#
|
202
|
+
# For proper referential integrity handling you must do the following:
|
203
|
+
# ALTER TABLE employees_of_company_1 add foreign key (company_id) references companies(id)
|
204
|
+
# ALTER TABLE employees_of_company_2 add foreign key (company_id) references companies(id)
|
205
|
+
# ALTER TABLE employees_of_company_3 add foreign key (company_id) references companies(id)
|
206
|
+
#
|
207
|
+
# ALTER TABLE employees_of_company_1 add foreign key (supervisor_id) references employees_of_company_1(id)
|
208
|
+
# ALTER TABLE employees_of_company_2 add foreign key (supervisor_id) references employees_of_company_2(id)
|
209
|
+
# ALTER TABLE employees_of_company_3 add foreign key (supervisor_id) references employees_of_company_3(id)
|
210
|
+
#
|
211
|
+
# The second set of alter tables brings up a good another consideration about postgres references and partitions.
|
212
|
+
# postgres will not follow references to a child table. So, a foreign key reference to "employees" in this
|
213
|
+
# set of alter statements would not work because postgres would expect the table "employees" to have
|
214
|
+
# the specific referenced record, but the record really exists in a child of employees. So, the alter statement
|
215
|
+
# forces the reference check on the specific child table we know must contain this employees supervisor (since
|
216
|
+
# such a supervisor would have to work for the same company in our model).
|
217
|
+
#
|
218
|
+
def add_references_to_partition_table(*partition_key_values)
|
219
|
+
configurator.foreign_keys(*partition_key_values).each do |foreign_key|
|
220
|
+
add_foreign_key(partition_table_name(*partition_key_values),
|
221
|
+
foreign_key.referencing_field,
|
222
|
+
foreign_key.referenced_table,
|
223
|
+
foreign_key.referenced_field)
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
##
|
228
|
+
# :method: connection
|
229
|
+
# delegated to the connection of the parent table class
|
230
|
+
|
231
|
+
##
|
232
|
+
# :method: execute
|
233
|
+
# delegated to the connection of the parent table class
|
234
|
+
|
235
|
+
##
|
236
|
+
# :method: create_schema
|
237
|
+
# delegated to the connection of the parent table class
|
238
|
+
|
239
|
+
##
|
240
|
+
# :method: drop_schema
|
241
|
+
# delegated to the connection of the parent table class
|
242
|
+
|
243
|
+
##
|
244
|
+
# :method: add_index
|
245
|
+
# delegated to the connection of the parent table class
|
246
|
+
|
247
|
+
##
|
248
|
+
# :method: remove_index
|
249
|
+
# delegated to the connection of the parent table class
|
250
|
+
|
251
|
+
##
|
252
|
+
# :method: transaction
|
253
|
+
# delegated to the connection of the parent table class
|
254
|
+
|
255
|
+
##
|
256
|
+
# :method: find_by_sql
|
257
|
+
# delegated to the connection of the parent table class
|
258
|
+
|
259
|
+
##
|
260
|
+
# :method: find
|
261
|
+
# delegated to the connection of the parent table class
|
262
|
+
|
263
|
+
extend Forwardable
|
264
|
+
def_delegators :parent_table_class, :connection, :find_by_sql, :transaction, :find, :configurator
|
265
|
+
def_delegators :connection, :execute, :add_index, :remove_index, :create_schema, :drop_schema, :add_foreign_key,
|
266
|
+
:create_table, :drop_table
|
267
|
+
end
|
268
|
+
end
|
269
|
+
end
|
@@ -93,7 +93,7 @@ module Partitioned
|
|
93
93
|
# Override this or order the tables from last (greatest value? greatest date?) to first.
|
94
94
|
#
|
95
95
|
def last_n_partitions_order_by_clause
|
96
|
-
return configurator.
|
96
|
+
return configurator.last_partitions_order_by_clause
|
97
97
|
end
|
98
98
|
|
99
99
|
#
|
@@ -110,10 +110,10 @@ module Partitioned
|
|
110
110
|
insert_redirector_name = parent_table_rule_name("insert", "redirector", *partition_key_values)
|
111
111
|
sql = <<-SQL
|
112
112
|
CREATE OR REPLACE RULE #{insert_redirector_name} AS
|
113
|
-
ON INSERT TO #{configurator.
|
113
|
+
ON INSERT TO #{configurator.table_name(*partition_key_values)}
|
114
114
|
DO INSTEAD
|
115
115
|
(
|
116
|
-
SELECT always_fail_on_insert('#{configurator.
|
116
|
+
SELECT always_fail_on_insert('#{configurator.table_name(*partition_key_values)}')
|
117
117
|
)
|
118
118
|
SQL
|
119
119
|
execute(sql)
|
@@ -141,7 +141,8 @@ module Partitioned
|
|
141
141
|
:id => false,
|
142
142
|
:options => "INHERITS (#{configurator.parent_table_name(*partition_key_values)})"
|
143
143
|
}) do |t|
|
144
|
-
|
144
|
+
constraint = configurator.check_constraint(*partition_key_values)
|
145
|
+
t.check_constraint constraint if constraint
|
145
146
|
end
|
146
147
|
end
|
147
148
|
|
@@ -171,7 +172,7 @@ module Partitioned
|
|
171
172
|
# Used when creating the name of a SQL rule.
|
172
173
|
#
|
173
174
|
def parent_table_rule_name(name, suffix = "rule", *partition_key_values)
|
174
|
-
return "#{configurator.
|
175
|
+
return "#{configurator.table_name(*partition_key_values).gsub(/[.]/, '_')}_#{name}_#{suffix}"
|
175
176
|
end
|
176
177
|
|
177
178
|
#
|
@@ -103,10 +103,23 @@ module Partitioned
|
|
103
103
|
#
|
104
104
|
# @return [{SqlAdapter}] the object used to create sql statements for this partitioned model
|
105
105
|
def self.sql_adapter
|
106
|
-
@sql_adapter
|
106
|
+
@sql_adapter ||= connection.partitioned_sql_adapter(self)
|
107
107
|
return @sql_adapter
|
108
108
|
end
|
109
|
+
|
110
|
+
def self.arel_table_from_key_values(partition_key_values, as = nil)
|
111
|
+
@arel_tables ||= {}
|
112
|
+
new_arel_table = @arel_tables[[partition_key_values, as]]
|
113
|
+
|
114
|
+
unless new_arel_table
|
115
|
+
arel_engine_hash = {:engine => self.arel_engine, :as => as}
|
116
|
+
new_arel_table = Arel::Table.new(self.partition_table_name(*partition_key_values), arel_engine_hash)
|
117
|
+
@arel_tables[[partition_key_values, as]] = new_arel_table
|
118
|
+
end
|
109
119
|
|
120
|
+
return new_arel_table
|
121
|
+
end
|
122
|
+
|
110
123
|
#
|
111
124
|
# In activerecord 3.0 we need to supply an Arel::Table for the key value(s) used
|
112
125
|
# to determine the specific child table to access.
|
@@ -115,13 +128,8 @@ module Partitioned
|
|
115
128
|
# @param [String] as (nil) the name of the table associated with this Arel::Table
|
116
129
|
# @return [Arel::Table] the generated Arel::Table
|
117
130
|
def self.dynamic_arel_table(values, as = nil)
|
118
|
-
@arel_tables ||= {}
|
119
131
|
key_values = self.partition_key_values(values)
|
120
|
-
|
121
|
-
arel_engine_hash = {:engine => self.arel_engine}
|
122
|
-
arel_engine_hash[:as] = as unless as.blank?
|
123
|
-
new_arel_table = Arel::Table.new(self.partition_table_name(*key_values), arel_engine_hash)
|
124
|
-
return new_arel_table
|
132
|
+
return arel_table_from_key_values(key_values, as)
|
125
133
|
end
|
126
134
|
|
127
135
|
#
|
@@ -131,9 +139,8 @@ module Partitioned
|
|
131
139
|
# @param [String] as (nil) the name of the table associated with the Arel::Table
|
132
140
|
# @return [Arel::Table] the generated Arel::Table
|
133
141
|
def dynamic_arel_table(as = nil)
|
134
|
-
|
135
|
-
|
136
|
-
return self.class.dynamic_arel_table(key_values, as)
|
142
|
+
key_values = self.class.partition_key_values(attributes)
|
143
|
+
return self.class.arel_table_from_key_values(key_values, as)
|
137
144
|
end
|
138
145
|
|
139
146
|
#
|
@@ -149,10 +156,9 @@ module Partitioned
|
|
149
156
|
#
|
150
157
|
# @param [*Array<Object>] partition_field the field values to partition on
|
151
158
|
# @return [Hash] the scoping
|
152
|
-
def self.from_partition(*
|
153
|
-
table_alias_name = partition_table_alias_name(*
|
154
|
-
|
155
|
-
tap{|relation| relation.table.table_alias = table_alias_name}
|
159
|
+
def self.from_partition(*partition_key_values)
|
160
|
+
table_alias_name = partition_table_alias_name(*partition_key_values)
|
161
|
+
return ActiveRecord::Relation.new(self, self.arel_table_from_key_values(partition_key_values, table_alias_name))
|
156
162
|
end
|
157
163
|
|
158
164
|
#
|
@@ -177,10 +183,8 @@ module Partitioned
|
|
177
183
|
#
|
178
184
|
# @param [*Array<Object>] partition_field the field values to partition on
|
179
185
|
# @return [Hash] the scoping
|
180
|
-
def self.from_partition_without_alias(*
|
181
|
-
|
182
|
-
from(table_alias_name).
|
183
|
-
tap{|relation| relation.table.table_alias = table_alias_name}
|
186
|
+
def self.from_partition_without_alias(*partition_key_values)
|
187
|
+
return ActiveRecord::Relation.new(self, self.arel_table_from_key_values(partition_key_values, nil))
|
184
188
|
end
|
185
189
|
|
186
190
|
#
|
@@ -305,7 +309,7 @@ module Partitioned
|
|
305
309
|
# A reasonable alias for this table
|
306
310
|
#
|
307
311
|
partition.table_alias_name lambda {|model, *partition_key_values|
|
308
|
-
return model.
|
312
|
+
return model.table_name
|
309
313
|
}
|
310
314
|
|
311
315
|
#
|
data/lib/partitioned/version.rb
CHANGED
data/lib/partitioned.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'monkey_patch_activerecord'
|
2
2
|
require 'monkey_patch_postgres'
|
3
|
+
require 'monkey_patch_redshift'
|
3
4
|
|
4
5
|
require 'partitioned/active_record_overrides'
|
5
6
|
require 'partitioned/partitioned_base/configurator.rb'
|
@@ -9,6 +10,7 @@ require 'partitioned/partitioned_base.rb'
|
|
9
10
|
require 'partitioned/partitioned_base/configurator/reader'
|
10
11
|
require 'partitioned/partitioned_base/partition_manager'
|
11
12
|
require 'partitioned/partitioned_base/sql_adapter'
|
13
|
+
require 'partitioned/partitioned_base/redshift_sql_adapter'
|
12
14
|
|
13
15
|
require 'partitioned/by_time_field'
|
14
16
|
require 'partitioned/by_yearly_time_field'
|
data/partitioned.gemspec
CHANGED
@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
|
|
7
7
|
s.name = 'partitioned'
|
8
8
|
s.version = Partitioned::VERSION
|
9
9
|
s.license = 'New BSD License'
|
10
|
-
s.date = '2013-
|
10
|
+
s.date = '2013-02-04'
|
11
11
|
s.summary = "Postgres table partitioning support for ActiveRecord."
|
12
12
|
s.description = "A gem providing support for table partitioning in ActiveRecord. Support is only available for postgres databases. Other features include child table management (creation and deletion) and bulk data creating and updating."
|
13
13
|
s.authors = ["Keith Gabryelski", "Aleksandr Dembskiy"]
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: partitioned
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.5
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2013-
|
13
|
+
date: 2013-02-04 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: pg
|
@@ -106,6 +106,7 @@ files:
|
|
106
106
|
- init.rb
|
107
107
|
- lib/monkey_patch_activerecord.rb
|
108
108
|
- lib/monkey_patch_postgres.rb
|
109
|
+
- lib/monkey_patch_redshift.rb
|
109
110
|
- lib/partitioned.rb
|
110
111
|
- lib/partitioned/active_record_overrides.rb
|
111
112
|
- lib/partitioned/by_created_at.rb
|
@@ -127,6 +128,7 @@ files:
|
|
127
128
|
- lib/partitioned/partitioned_base/configurator/dsl.rb
|
128
129
|
- lib/partitioned/partitioned_base/configurator/reader.rb
|
129
130
|
- lib/partitioned/partitioned_base/partition_manager.rb
|
131
|
+
- lib/partitioned/partitioned_base/redshift_sql_adapter.rb
|
130
132
|
- lib/partitioned/partitioned_base/sql_adapter.rb
|
131
133
|
- lib/partitioned/version.rb
|
132
134
|
- lib/tasks/desirable_tasks.rake
|
@@ -201,7 +203,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
201
203
|
version: '0'
|
202
204
|
requirements: []
|
203
205
|
rubyforge_project:
|
204
|
-
rubygems_version: 1.8.
|
206
|
+
rubygems_version: 1.8.25
|
205
207
|
signing_key:
|
206
208
|
specification_version: 3
|
207
209
|
summary: Postgres table partitioning support for ActiveRecord.
|