partitioned 1.1.3 → 1.1.5
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile.lock +6 -6
- data/LICENSE +1 -1
- data/lib/monkey_patch_activerecord.rb +12 -7
- data/lib/monkey_patch_postgres.rb +3 -0
- data/lib/monkey_patch_redshift.rb +114 -0
- data/lib/partitioned/multi_level/configurator/reader.rb +39 -21
- data/lib/partitioned/multi_level/partition_manager.rb +2 -2
- data/lib/partitioned/partitioned_base/configurator/data.rb +4 -1
- data/lib/partitioned/partitioned_base/configurator/dsl.rb +10 -0
- data/lib/partitioned/partitioned_base/configurator/reader.rb +10 -1
- data/lib/partitioned/partitioned_base/partition_manager.rb +1 -0
- data/lib/partitioned/partitioned_base/redshift_sql_adapter.rb +269 -0
- data/lib/partitioned/partitioned_base/sql_adapter.rb +6 -5
- data/lib/partitioned/partitioned_base.rb +23 -19
- data/lib/partitioned/version.rb +1 -1
- data/lib/partitioned.rb +2 -0
- data/partitioned.gemspec +1 -1
- metadata +5 -3
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
partitioned (1.1.
|
4
|
+
partitioned (1.1.4)
|
5
5
|
bulk_data_methods (= 1.0.0)
|
6
6
|
pg
|
7
7
|
rails (>= 3.2.8)
|
@@ -48,7 +48,7 @@ GEM
|
|
48
48
|
hike (1.2.1)
|
49
49
|
i18n (0.6.1)
|
50
50
|
journey (1.0.4)
|
51
|
-
jquery-rails (2.
|
51
|
+
jquery-rails (2.2.0)
|
52
52
|
railties (>= 3.0, < 5.0)
|
53
53
|
thor (>= 0.14, < 2.0)
|
54
54
|
json (1.7.6)
|
@@ -56,14 +56,14 @@ GEM
|
|
56
56
|
i18n (>= 0.4.0)
|
57
57
|
mime-types (~> 1.16)
|
58
58
|
treetop (~> 1.4.8)
|
59
|
-
mime-types (1.
|
59
|
+
mime-types (1.20.1)
|
60
60
|
multi_json (1.5.0)
|
61
61
|
pg (0.14.1)
|
62
62
|
polyglot (0.3.3)
|
63
63
|
rack (1.4.4)
|
64
64
|
rack-cache (1.2)
|
65
65
|
rack (>= 0.4)
|
66
|
-
rack-ssl (1.3.
|
66
|
+
rack-ssl (1.3.3)
|
67
67
|
rack
|
68
68
|
rack-test (0.6.2)
|
69
69
|
rack (>= 1.0)
|
@@ -88,7 +88,7 @@ GEM
|
|
88
88
|
rspec-core (2.12.2)
|
89
89
|
rspec-expectations (2.12.1)
|
90
90
|
diff-lcs (~> 1.1.3)
|
91
|
-
rspec-mocks (2.12.
|
91
|
+
rspec-mocks (2.12.2)
|
92
92
|
rspec-rails (2.12.2)
|
93
93
|
actionpack (>= 3.0)
|
94
94
|
activesupport (>= 3.0)
|
@@ -101,7 +101,7 @@ GEM
|
|
101
101
|
multi_json (~> 1.0)
|
102
102
|
rack (~> 1.0)
|
103
103
|
tilt (~> 1.1, != 1.3.0)
|
104
|
-
thor (0.
|
104
|
+
thor (0.17.0)
|
105
105
|
tilt (1.3.3)
|
106
106
|
treetop (1.4.12)
|
107
107
|
polyglot
|
data/LICENSE
CHANGED
@@ -17,21 +17,26 @@ module ActiveRecord
|
|
17
17
|
# that no changes should be made (since they can't be persisted).
|
18
18
|
def destroy
|
19
19
|
destroy_associations
|
20
|
-
|
20
|
+
|
21
21
|
if persisted?
|
22
22
|
IdentityMap.remove(self) if IdentityMap.enabled?
|
23
23
|
pk = self.class.primary_key
|
24
24
|
column = self.class.columns_hash[pk]
|
25
25
|
substitute = connection.substitute_at(column, 0)
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
26
|
+
|
27
|
+
if self.class.respond_to?(:dynamic_arel_table)
|
28
|
+
using_arel_table = dynamic_arel_table()
|
29
|
+
relation = ActiveRecord::Relation.new(self.class, using_arel_table).
|
30
|
+
where(using_arel_table[pk].eq(substitute))
|
31
|
+
else
|
32
|
+
using_arel_table = self.class.arel_table
|
33
|
+
relation = self.class.unscoped.where(using_arel_table[pk].eq(substitute))
|
34
|
+
end
|
35
|
+
|
31
36
|
relation.bind_values = [[column, id]]
|
32
37
|
relation.delete_all
|
33
38
|
end
|
34
|
-
|
39
|
+
|
35
40
|
@destroyed = true
|
36
41
|
freeze
|
37
42
|
end
|
@@ -29,6 +29,9 @@ module ActiveRecord::ConnectionAdapters
|
|
29
29
|
# to take advantage of these SQL builders.
|
30
30
|
#
|
31
31
|
class PostgreSQLAdapter < AbstractAdapter
|
32
|
+
def partitioned_sql_adapter(model)
|
33
|
+
return Partitioned::PartitionedBase::SqlAdapter.new(model)
|
34
|
+
end
|
32
35
|
|
33
36
|
#
|
34
37
|
# Returns the sequence name for a table's primary key or some other specified key.
|
@@ -0,0 +1,114 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'active_record/base'
|
3
|
+
require 'active_record/connection_adapters/abstract_adapter'
|
4
|
+
require 'active_record/connection_adapters/redshift_adapter'
|
5
|
+
|
6
|
+
#
|
7
|
+
# Patching {ActiveRecord::ConnectionAdapters::TableDefinition} and
|
8
|
+
# {ActiveRecord::ConnectionAdapters::PostgreSQLAdapter} to add functionality
|
9
|
+
# needed to abstract partition specific SQL statements.
|
10
|
+
#
|
11
|
+
module ActiveRecord::ConnectionAdapters
|
12
|
+
#
|
13
|
+
# Patches extending the postgres adapter with new operations for managing
|
14
|
+
# sequences (and sets of sequence values), schemas and foreign keys.
|
15
|
+
# These should go into AbstractAdapter allowing any database adapter
|
16
|
+
# to take advantage of these SQL builders.
|
17
|
+
#
|
18
|
+
class RedshiftAdapter < AbstractAdapter
|
19
|
+
def partitioned_sql_adapter(model)
|
20
|
+
return Partitioned::PartitionedBase::RedshiftSqlAdapter.new(model)
|
21
|
+
end
|
22
|
+
|
23
|
+
#
|
24
|
+
# Returns the sequence name for a table's primary key or some other specified key.
|
25
|
+
#
|
26
|
+
# the default version strips off the schema name on the table (if it exists), as:
|
27
|
+
# serial_sequence(table_name, pk || 'id').split('.').last
|
28
|
+
# i can't see any good reason for that -- in fact, it seems completely
|
29
|
+
# broken -- if you have a table public.foos and other.foos, you'll fail to
|
30
|
+
# get the correct schema if you fetch the default schema name from model
|
31
|
+
# associated with other.foos
|
32
|
+
#
|
33
|
+
def default_sequence_name(table_name, pk = nil) #:nodoc:
|
34
|
+
serial_sequence(table_name, pk || 'id')
|
35
|
+
rescue ActiveRecord::StatementInvalid => e
|
36
|
+
"#{table_name}_#{pk || 'id'}_seq"
|
37
|
+
end
|
38
|
+
|
39
|
+
#
|
40
|
+
# Get the next value in a sequence. Used on INSERT operation for
|
41
|
+
# partitioning like by_id because the ID is required before the insert
|
42
|
+
# so that the specific child table is known ahead of time.
|
43
|
+
#
|
44
|
+
# @param [String] sequence_name the name of the sequence to fetch the next value from
|
45
|
+
# @return [Integer] the value from the sequence
|
46
|
+
def next_sequence_value(sequence_name)
|
47
|
+
return execute("select nextval('#{sequence_name}')").field_values("nextval").first.to_i
|
48
|
+
end
|
49
|
+
|
50
|
+
#
|
51
|
+
# Get the some next values in a sequence.
|
52
|
+
#
|
53
|
+
# @param [String] sequence_name the name of the sequence to fetch the next values from
|
54
|
+
# @param [Integer] batch_size count of values.
|
55
|
+
# @return [Array<Integer>] an array of values from the sequence
|
56
|
+
def next_sequence_values(sequence_name, batch_size)
|
57
|
+
result = execute("select nextval('#{sequence_name}') from generate_series(1, #{batch_size})")
|
58
|
+
return result.field_values("nextval").map(&:to_i)
|
59
|
+
end
|
60
|
+
|
61
|
+
#
|
62
|
+
# Causes active resource to fetch the primary key for the table (using next_sequence_value())
|
63
|
+
# just before an insert. We need the prefetch to happen but we don't have enough information
|
64
|
+
# here to determine if it should happen, so Relation::insert has been modified to request of
|
65
|
+
# the ActiveRecord::Base derived class if it requires a prefetch.
|
66
|
+
#
|
67
|
+
# @param [String] table_name the table name to query
|
68
|
+
# @return [Boolean] returns true if the table should have its primary key prefetched.
|
69
|
+
def prefetch_primary_key?(table_name)
|
70
|
+
return false
|
71
|
+
end
|
72
|
+
|
73
|
+
#
|
74
|
+
# Creates a schema given a name.
|
75
|
+
#
|
76
|
+
# @param [String] name the name of the schema.
|
77
|
+
# @param [Hash] options ({}) options for creating a schema
|
78
|
+
# @option options [Boolean] :unless_exists (false) check if schema exists.
|
79
|
+
# @return [optional] undefined
|
80
|
+
def create_schema(name, options = {})
|
81
|
+
if options[:unless_exists]
|
82
|
+
return if execute("select count(*) from pg_namespace where nspname = '#{name}'").getvalue(0,0).to_i > 0
|
83
|
+
end
|
84
|
+
execute("CREATE SCHEMA #{name}")
|
85
|
+
end
|
86
|
+
|
87
|
+
#
|
88
|
+
# Drop a schema given a name.
|
89
|
+
#
|
90
|
+
# @param [String] name the name of the schema.
|
91
|
+
# @param [Hash] options ({}) options for dropping a schema
|
92
|
+
# @option options [Boolean] :if_exists (false) check if schema exists.
|
93
|
+
# @option options [Boolean] :cascade (false) drop dependant objects
|
94
|
+
# @return [optional] undefined
|
95
|
+
def drop_schema(name, options = {})
|
96
|
+
if options[:if_exists]
|
97
|
+
return if execute("select count(*) from pg_namespace where nspname = '#{name}'").getvalue(0,0).to_i == 0
|
98
|
+
end
|
99
|
+
execute("DROP SCHEMA #{name}#{' cascade' if options[:cascade]}")
|
100
|
+
end
|
101
|
+
|
102
|
+
#
|
103
|
+
# Add foreign key constraint to table.
|
104
|
+
#
|
105
|
+
# @param [String] referencing_table_name the name of the table containing the foreign key
|
106
|
+
# @param [String] referencing_field_name the name of foreign key column
|
107
|
+
# @param [String] referenced_table_name the name of the table referenced by the foreign key
|
108
|
+
# @param [String] referenced_field_name (:id) the name of the column referenced by the foreign key
|
109
|
+
# @return [optional] undefined
|
110
|
+
def add_foreign_key(referencing_table_name, referencing_field_name, referenced_table_name, referenced_field_name = :id)
|
111
|
+
execute("ALTER TABLE #{referencing_table_name} add foreign key (#{referencing_field_name}) references #{referenced_table_name}(#{referenced_field_name})")
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -5,6 +5,10 @@ module Partitioned
|
|
5
5
|
# {PartitionManager} to request partitioning information froma
|
6
6
|
# centralized source from multi level partitioned models
|
7
7
|
class Reader < Partitioned::PartitionedBase::Configurator::Reader
|
8
|
+
|
9
|
+
alias :base_collect_from_collection :collect_from_collection
|
10
|
+
alias :base_collect :collect
|
11
|
+
|
8
12
|
# configurator for a specific class level
|
9
13
|
UsingConfigurator = Struct.new(:model, :sliced_class, :dsl)
|
10
14
|
|
@@ -20,7 +24,7 @@ module Partitioned
|
|
20
24
|
# @return [Array<Symbol>] fields used to partition this model
|
21
25
|
def on_fields
|
22
26
|
unless @on_fields
|
23
|
-
@on_fields =
|
27
|
+
@on_fields = collect(&:on_field).map(&:to_sym)
|
24
28
|
end
|
25
29
|
return @on_fields
|
26
30
|
end
|
@@ -67,6 +71,35 @@ module Partitioned
|
|
67
71
|
return using_configurator(index).check_constraint(value)
|
68
72
|
end
|
69
73
|
|
74
|
+
def indexes(*partition_key_values)
|
75
|
+
bag = {}
|
76
|
+
partition_key_values.each_with_index do |key_value, index|
|
77
|
+
bag.merge!(using_configurator(index).indexes(key_value))
|
78
|
+
end
|
79
|
+
base_collect_from_collection(*partition_key_values, &:indexes).inject(bag) do |bag, data_index|
|
80
|
+
bag[data_index.field] = (data_index.options || {}) unless data_index.blank?
|
81
|
+
bag
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
#
|
86
|
+
# Foreign keys to create on each leaf partition.
|
87
|
+
#
|
88
|
+
def foreign_keys(*partition_key_values)
|
89
|
+
set = Set.new
|
90
|
+
partition_key_values.each_with_index do |key_value, index|
|
91
|
+
set.merge(using_configurator(index).foreign_keys(key_value))
|
92
|
+
end
|
93
|
+
base_collect_from_collection(*partition_key_values, &:foreign_keys).inject(set) do |set, new_items|
|
94
|
+
if new_items.is_a? Array
|
95
|
+
set += new_items
|
96
|
+
else
|
97
|
+
set += [new_items]
|
98
|
+
end
|
99
|
+
set
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
70
103
|
#
|
71
104
|
# The name of the child table without the schema name or name prefix.
|
72
105
|
#
|
@@ -117,14 +150,14 @@ module Partitioned
|
|
117
150
|
|
118
151
|
def using_classes
|
119
152
|
unless @using_classes
|
120
|
-
@using_classes =
|
153
|
+
@using_classes = base_collect_from_collection(&:using_classes).inject([]) do |array,new_items|
|
121
154
|
array += [*new_items]
|
122
155
|
end.to_a
|
123
156
|
end
|
124
157
|
return @using_classes
|
125
158
|
end
|
126
159
|
|
127
|
-
def
|
160
|
+
def collect(*partition_key_values, &block)
|
128
161
|
values = []
|
129
162
|
using_configurators.each do |using_configurator|
|
130
163
|
data = using_configurator.dsl.data
|
@@ -137,25 +170,10 @@ module Partitioned
|
|
137
170
|
values << intermediate_value unless intermediate_value.blank?
|
138
171
|
end
|
139
172
|
end
|
140
|
-
return values
|
141
|
-
end
|
142
|
-
|
143
|
-
def using_collect_first(*partition_key_values, &block)
|
144
|
-
using_configurators.each do |using_configurator|
|
145
|
-
data = using_configurator.dsl.data
|
146
|
-
intermediate_value = block.call(data) rescue nil
|
147
|
-
if intermediate_value.is_a? Proc
|
148
|
-
return intermediate_value.call(using_configurator.model, *partition_key_values)
|
149
|
-
elsif intermediate_value.is_a? String
|
150
|
-
return eval("\"#{intermediate_value}\"")
|
151
|
-
else
|
152
|
-
return intermediate_value unless intermediate_value.nil?
|
153
|
-
end
|
154
|
-
end
|
155
|
-
return nil
|
173
|
+
return base_collect(*partition_key_values, &block) + values
|
156
174
|
end
|
157
175
|
|
158
|
-
def
|
176
|
+
def collect_from_collection(*partition_key_values, &block)
|
159
177
|
values = []
|
160
178
|
using_configurators.each do |using_configurator|
|
161
179
|
data = using_configurator.dsl.data
|
@@ -171,7 +189,7 @@ module Partitioned
|
|
171
189
|
end
|
172
190
|
end
|
173
191
|
end
|
174
|
-
return values
|
192
|
+
return base_collect_from_collection(*partition_key_values, &block) + values
|
175
193
|
end
|
176
194
|
|
177
195
|
end
|
@@ -17,8 +17,6 @@ module Partitioned
|
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
|
-
protected
|
21
|
-
|
22
20
|
#
|
23
21
|
# Create a specific child table that does not currently
|
24
22
|
# exist and whose schema (the schema that the table exists in)
|
@@ -37,6 +35,8 @@ module Partitioned
|
|
37
35
|
end
|
38
36
|
end
|
39
37
|
|
38
|
+
protected
|
39
|
+
|
40
40
|
#
|
41
41
|
# Is the table a child table without itself having any children.
|
42
42
|
# generally leaf tables are where all indexes and foreign key
|
@@ -42,7 +42,8 @@ module Partitioned
|
|
42
42
|
:schema_name, :name_prefix, :base_name,
|
43
43
|
:part_name, :table_name, :table_alias_name, :parent_table_schema_name,
|
44
44
|
:parent_table_name, :check_constraint, :encoded_name,
|
45
|
-
:janitorial_creates_needed, :janitorial_archives_needed, :janitorial_drops_needed
|
45
|
+
:janitorial_creates_needed, :janitorial_archives_needed, :janitorial_drops_needed,
|
46
|
+
:after_partition_table_create_hooks
|
46
47
|
|
47
48
|
def initialize
|
48
49
|
@on_field = nil
|
@@ -70,6 +71,8 @@ module Partitioned
|
|
70
71
|
@janitorial_creates_needed = nil
|
71
72
|
@janitorial_archives_needed = nil
|
72
73
|
@janitorial_drops_needed = nil
|
74
|
+
|
75
|
+
@after_partition_table_create_hooks = []
|
73
76
|
end
|
74
77
|
end
|
75
78
|
end
|
@@ -263,6 +263,16 @@ module Partitioned
|
|
263
263
|
end
|
264
264
|
end
|
265
265
|
|
266
|
+
def after_partition_table_create_hook(method_name)
|
267
|
+
if method_name.is_a? Proc
|
268
|
+
data.after_partition_table_create_hooks << method_name
|
269
|
+
else
|
270
|
+
# XXX should be a symbol
|
271
|
+
data.after_partition_table_create_hooks << lambda {|model,*partition_key_values| model.send(method_name, *partition_key_values)}
|
272
|
+
end
|
273
|
+
end
|
274
|
+
|
275
|
+
|
266
276
|
#
|
267
277
|
# Define the check constraint for a given child table.
|
268
278
|
#
|
@@ -30,6 +30,7 @@ module Partitioned
|
|
30
30
|
@janitorial_creates_needed = nil
|
31
31
|
@janitorial_archives_needed = nil
|
32
32
|
@janitorial_drops_needed = nil
|
33
|
+
@after_partition_table_create_hooks = nil
|
33
34
|
end
|
34
35
|
|
35
36
|
#
|
@@ -101,7 +102,11 @@ module Partitioned
|
|
101
102
|
# The full name of a child table defined by the partition key values.
|
102
103
|
#
|
103
104
|
def table_name(*partition_key_values)
|
104
|
-
|
105
|
+
if partition_key_values.length < 1
|
106
|
+
return model.table_name
|
107
|
+
else
|
108
|
+
return collect_first(*partition_key_values, &:table_name)
|
109
|
+
end
|
105
110
|
end
|
106
111
|
|
107
112
|
#
|
@@ -167,6 +172,10 @@ module Partitioned
|
|
167
172
|
return @janitorial_drops_needed
|
168
173
|
end
|
169
174
|
|
175
|
+
def run_after_partition_table_create_hooks(*partition_key_values)
|
176
|
+
collect_from_collection(*partition_key_values, &:after_partition_table_create_hooks)
|
177
|
+
end
|
178
|
+
|
170
179
|
protected
|
171
180
|
|
172
181
|
def configurators
|
@@ -127,6 +127,7 @@ module Partitioned
|
|
127
127
|
create_partition_table(*partition_key_values)
|
128
128
|
add_partition_table_index(*partition_key_values)
|
129
129
|
add_references_to_partition_table(*partition_key_values)
|
130
|
+
configurator.run_after_partition_table_create_hooks(*partition_key_values)
|
130
131
|
end
|
131
132
|
|
132
133
|
##
|
@@ -0,0 +1,269 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
3
|
+
module Partitioned
|
4
|
+
class PartitionedBase
|
5
|
+
#
|
6
|
+
# SqlAdapter
|
7
|
+
# manages requests of partitioned tables.
|
8
|
+
#
|
9
|
+
class RedshiftSqlAdapter
|
10
|
+
attr_reader :parent_table_class
|
11
|
+
|
12
|
+
def initialize(parent_table_class)
|
13
|
+
@parent_table_class = parent_table_class
|
14
|
+
end
|
15
|
+
|
16
|
+
#
|
17
|
+
# Ensure our function for warning about improper partition usage is in place.
|
18
|
+
#
|
19
|
+
# Name: always_fail_on_insert(text); Type: FUNCTION; Schema: public
|
20
|
+
#
|
21
|
+
# Used to raise an exception explaining why a specific insert (into a parent
|
22
|
+
# table which should never have records) should never be attempted.
|
23
|
+
#
|
24
|
+
def ensure_always_fail_on_insert_exists
|
25
|
+
# XXX nothing can be done here
|
26
|
+
end
|
27
|
+
|
28
|
+
#
|
29
|
+
# Child tables whose parent table is 'foos', typically exist in a schema named foos_partitions.
|
30
|
+
#
|
31
|
+
# *partition_key_values are needed here to support the use of multiple schemas to keep tables in.
|
32
|
+
#
|
33
|
+
def create_partition_schema(*partition_key_values)
|
34
|
+
create_schema(configurator.schema_name, :unless_exists => true)
|
35
|
+
end
|
36
|
+
|
37
|
+
#
|
38
|
+
# Does a specific child partition exist.
|
39
|
+
#
|
40
|
+
def partition_exists?(*partition_key_values)
|
41
|
+
return find(:first,
|
42
|
+
:from => "pg_tables",
|
43
|
+
:select => "count(*) as count",
|
44
|
+
:conditions => ["schemaname = ? and tablename = ?",
|
45
|
+
configurator.schema_name,
|
46
|
+
configurator.part_name(*partition_key_values)
|
47
|
+
]).count.to_i == 1
|
48
|
+
end
|
49
|
+
|
50
|
+
#
|
51
|
+
# Returns an array of partition table names from last to first limited to
|
52
|
+
# the number of entries requested by its first parameter.
|
53
|
+
#
|
54
|
+
# The magic here is in the overridden method "last_n_partitions_order_by_clause"
|
55
|
+
# which is designed to order a list of partition table names (table names without
|
56
|
+
# their schema name) from last to first.
|
57
|
+
#
|
58
|
+
# If the child table names are the format "pYYYYMMDD" where YYYY is a four digit year, MM is
|
59
|
+
# a month number and DD is a day number, you would use the following to order from last to
|
60
|
+
# first:
|
61
|
+
# tablename desc
|
62
|
+
#
|
63
|
+
# For child table names of the format "pXXXX" where XXXX is a number, you may want something like:
|
64
|
+
# substring(tablename, 2)::integer desc
|
65
|
+
#
|
66
|
+
# For clarity, the sql executed is:
|
67
|
+
# select tablename from pg_tables where schemaname = $1 order by $2 limit $3
|
68
|
+
# where:
|
69
|
+
# $1 = the name of schema (foos_partitions)
|
70
|
+
# $2 = the order by clause that would make the greatest table name listed first
|
71
|
+
# $3 = the parameter 'how_many'
|
72
|
+
#
|
73
|
+
def last_n_partition_names(how_many = 1)
|
74
|
+
return find(:all,
|
75
|
+
:from => "pg_tables",
|
76
|
+
:select => :tablename,
|
77
|
+
:conditions => ["schemaname = ?", configurator.schema_name],
|
78
|
+
:order => last_n_partitions_order_by_clause,
|
79
|
+
:limit => how_many).map(&:tablename)
|
80
|
+
end
|
81
|
+
|
82
|
+
#
|
83
|
+
# Override this or order the tables from last (greatest value? greatest date?) to first.
|
84
|
+
#
|
85
|
+
def last_n_partitions_order_by_clause
|
86
|
+
return configurator.last_partitions_order_by_clause
|
87
|
+
end
|
88
|
+
|
89
|
+
#
|
90
|
+
# Used to create the parent table rule to ensure.
|
91
|
+
#
|
92
|
+
# This will cause an error on attempt to insert into the parent table.
|
93
|
+
#
|
94
|
+
# We want all records to exist in one of the child tables so the
|
95
|
+
# query planner can optimize access to the records.
|
96
|
+
#
|
97
|
+
def add_parent_table_rules(*partition_key_values)
|
98
|
+
# XXX nothing can be done here
|
99
|
+
end
|
100
|
+
|
101
|
+
#
|
102
|
+
# The name of the table (schemaname.childtablename) given the check constraint values.
|
103
|
+
#
|
104
|
+
def partition_table_name(*partition_key_values)
|
105
|
+
return configurator.table_name(*partition_key_values)
|
106
|
+
end
|
107
|
+
|
108
|
+
#
|
109
|
+
# A reasonable alias for the partition table
|
110
|
+
#
|
111
|
+
def partition_table_alias_name(*partition_key_values)
|
112
|
+
return configurator.table_alias_name(*partition_key_values)
|
113
|
+
end
|
114
|
+
|
115
|
+
#
|
116
|
+
# Create a single child table.
|
117
|
+
#
|
118
|
+
def create_partition_table(*partition_key_values)
|
119
|
+
execute("select * into #{configurator.table_name(*partition_key_values)} from #{configurator.parent_table_name(*partition_key_values)} where false")
|
120
|
+
end
|
121
|
+
|
122
|
+
#
|
123
|
+
# Remove a specific single child table.
|
124
|
+
#
|
125
|
+
def drop_partition_table(*partition_key_values)
|
126
|
+
drop_table(configurator.table_name(*partition_key_values))
|
127
|
+
end
|
128
|
+
|
129
|
+
#
|
130
|
+
# Add indexes that must exist on child tables. Only leaf child tables
|
131
|
+
# need indexes as parent table indexes are not used in postgres.
|
132
|
+
#
|
133
|
+
def add_partition_table_index(*partition_key_values)
|
134
|
+
configurator.indexes(*partition_key_values).each do |field,options|
|
135
|
+
used_options = options.clone
|
136
|
+
unless used_options.has_key?(:name)
|
137
|
+
name = [*field].join('_')
|
138
|
+
used_options[:name] = used_options[:unique] ? unique_index_name(name, *partition_key_values) : index_name(name, *partition_key_values)
|
139
|
+
end
|
140
|
+
add_index(partition_table_name(*partition_key_values), field, used_options)
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
#
|
145
|
+
# Used when creating the name of a SQL rule.
|
146
|
+
#
|
147
|
+
def parent_table_rule_name(name, suffix = "rule", *partition_key_values)
|
148
|
+
return "#{configurator.table_name(*partition_key_values).gsub(/[.]/, '_')}_#{name}_#{suffix}"
|
149
|
+
end
|
150
|
+
|
151
|
+
#
|
152
|
+
# Used to create index names.
|
153
|
+
#
|
154
|
+
def index_name(name, *partition_key_values)
|
155
|
+
return "#{configurator.part_name(*partition_key_values)}_#{name}_idx"
|
156
|
+
end
|
157
|
+
|
158
|
+
#
|
159
|
+
# Used to create index names.
|
160
|
+
#
|
161
|
+
def unique_index_name(name, *partition_key_values)
|
162
|
+
return "#{configurator.part_name(*partition_key_values)}_#{name}_udx"
|
163
|
+
end
|
164
|
+
|
165
|
+
#
|
166
|
+
# This is here for derived classes to set up references to added columns
|
167
|
+
# (or columns in the parent that need foreign key constraints).
|
168
|
+
#
|
169
|
+
# Foreign keys are not inherited in postgres. So, a parent table
|
170
|
+
# of the form:
|
171
|
+
#
|
172
|
+
# -- this is the referenced table
|
173
|
+
# create table companies
|
174
|
+
# (
|
175
|
+
# id serial not null primary key,
|
176
|
+
# created_at timestamp not null default now(),
|
177
|
+
# updated_at timestamp,
|
178
|
+
# name text not null
|
179
|
+
# );
|
180
|
+
#
|
181
|
+
# -- this is the parent table
|
182
|
+
# create table employees
|
183
|
+
# (
|
184
|
+
# id serial not null primary key,
|
185
|
+
# created_at timestamp not null default now(),
|
186
|
+
# updated_at timestamp,
|
187
|
+
# name text not null,
|
188
|
+
# company_id integer not null references companies,
|
189
|
+
# supervisor_id integer not null references employees
|
190
|
+
# );
|
191
|
+
#
|
192
|
+
# -- some children
|
193
|
+
# create table employees_of_company_1 ( CHECK ( company_id = 1 ) ) INHERITS (employees);
|
194
|
+
# create table employees_of_company_2 ( CHECK ( company_id = 2 ) ) INHERITS (employees);
|
195
|
+
# create table employees_of_company_3 ( CHECK ( company_id = 3 ) ) INHERITS (employees);
|
196
|
+
#
|
197
|
+
# Since postgres does not inherit referential integrity from parent tables, the following
|
198
|
+
# insert will work:
|
199
|
+
# insert into employees_of_company_1 (name, company_id, supervisor_id) values ('joe', 1, 10);
|
200
|
+
# even if there is no record in companies with id = 1 and there is no record in employees with id = 10
|
201
|
+
#
|
202
|
+
# For proper referential integrity handling you must do the following:
|
203
|
+
# ALTER TABLE employees_of_company_1 add foreign key (company_id) references companies(id)
|
204
|
+
# ALTER TABLE employees_of_company_2 add foreign key (company_id) references companies(id)
|
205
|
+
# ALTER TABLE employees_of_company_3 add foreign key (company_id) references companies(id)
|
206
|
+
#
|
207
|
+
# ALTER TABLE employees_of_company_1 add foreign key (supervisor_id) references employees_of_company_1(id)
|
208
|
+
# ALTER TABLE employees_of_company_2 add foreign key (supervisor_id) references employees_of_company_2(id)
|
209
|
+
# ALTER TABLE employees_of_company_3 add foreign key (supervisor_id) references employees_of_company_3(id)
|
210
|
+
#
|
211
|
+
# The second set of alter tables brings up a good another consideration about postgres references and partitions.
|
212
|
+
# postgres will not follow references to a child table. So, a foreign key reference to "employees" in this
|
213
|
+
# set of alter statements would not work because postgres would expect the table "employees" to have
|
214
|
+
# the specific referenced record, but the record really exists in a child of employees. So, the alter statement
|
215
|
+
# forces the reference check on the specific child table we know must contain this employees supervisor (since
|
216
|
+
# such a supervisor would have to work for the same company in our model).
|
217
|
+
#
|
218
|
+
def add_references_to_partition_table(*partition_key_values)
|
219
|
+
configurator.foreign_keys(*partition_key_values).each do |foreign_key|
|
220
|
+
add_foreign_key(partition_table_name(*partition_key_values),
|
221
|
+
foreign_key.referencing_field,
|
222
|
+
foreign_key.referenced_table,
|
223
|
+
foreign_key.referenced_field)
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
##
|
228
|
+
# :method: connection
|
229
|
+
# delegated to the connection of the parent table class
|
230
|
+
|
231
|
+
##
|
232
|
+
# :method: execute
|
233
|
+
# delegated to the connection of the parent table class
|
234
|
+
|
235
|
+
##
|
236
|
+
# :method: create_schema
|
237
|
+
# delegated to the connection of the parent table class
|
238
|
+
|
239
|
+
##
|
240
|
+
# :method: drop_schema
|
241
|
+
# delegated to the connection of the parent table class
|
242
|
+
|
243
|
+
##
|
244
|
+
# :method: add_index
|
245
|
+
# delegated to the connection of the parent table class
|
246
|
+
|
247
|
+
##
|
248
|
+
# :method: remove_index
|
249
|
+
# delegated to the connection of the parent table class
|
250
|
+
|
251
|
+
##
|
252
|
+
# :method: transaction
|
253
|
+
# delegated to the connection of the parent table class
|
254
|
+
|
255
|
+
##
|
256
|
+
# :method: find_by_sql
|
257
|
+
# delegated to the connection of the parent table class
|
258
|
+
|
259
|
+
##
|
260
|
+
# :method: find
|
261
|
+
# delegated to the connection of the parent table class
|
262
|
+
|
263
|
+
extend Forwardable
|
264
|
+
def_delegators :parent_table_class, :connection, :find_by_sql, :transaction, :find, :configurator
|
265
|
+
def_delegators :connection, :execute, :add_index, :remove_index, :create_schema, :drop_schema, :add_foreign_key,
|
266
|
+
:create_table, :drop_table
|
267
|
+
end
|
268
|
+
end
|
269
|
+
end
|
@@ -93,7 +93,7 @@ module Partitioned
|
|
93
93
|
# Override this or order the tables from last (greatest value? greatest date?) to first.
|
94
94
|
#
|
95
95
|
def last_n_partitions_order_by_clause
|
96
|
-
return configurator.
|
96
|
+
return configurator.last_partitions_order_by_clause
|
97
97
|
end
|
98
98
|
|
99
99
|
#
|
@@ -110,10 +110,10 @@ module Partitioned
|
|
110
110
|
insert_redirector_name = parent_table_rule_name("insert", "redirector", *partition_key_values)
|
111
111
|
sql = <<-SQL
|
112
112
|
CREATE OR REPLACE RULE #{insert_redirector_name} AS
|
113
|
-
ON INSERT TO #{configurator.
|
113
|
+
ON INSERT TO #{configurator.table_name(*partition_key_values)}
|
114
114
|
DO INSTEAD
|
115
115
|
(
|
116
|
-
SELECT always_fail_on_insert('#{configurator.
|
116
|
+
SELECT always_fail_on_insert('#{configurator.table_name(*partition_key_values)}')
|
117
117
|
)
|
118
118
|
SQL
|
119
119
|
execute(sql)
|
@@ -141,7 +141,8 @@ module Partitioned
|
|
141
141
|
:id => false,
|
142
142
|
:options => "INHERITS (#{configurator.parent_table_name(*partition_key_values)})"
|
143
143
|
}) do |t|
|
144
|
-
|
144
|
+
constraint = configurator.check_constraint(*partition_key_values)
|
145
|
+
t.check_constraint constraint if constraint
|
145
146
|
end
|
146
147
|
end
|
147
148
|
|
@@ -171,7 +172,7 @@ module Partitioned
|
|
171
172
|
# Used when creating the name of a SQL rule.
|
172
173
|
#
|
173
174
|
def parent_table_rule_name(name, suffix = "rule", *partition_key_values)
|
174
|
-
return "#{configurator.
|
175
|
+
return "#{configurator.table_name(*partition_key_values).gsub(/[.]/, '_')}_#{name}_#{suffix}"
|
175
176
|
end
|
176
177
|
|
177
178
|
#
|
@@ -103,10 +103,23 @@ module Partitioned
|
|
103
103
|
#
|
104
104
|
# @return [{SqlAdapter}] the object used to create sql statements for this partitioned model
|
105
105
|
def self.sql_adapter
|
106
|
-
@sql_adapter
|
106
|
+
@sql_adapter ||= connection.partitioned_sql_adapter(self)
|
107
107
|
return @sql_adapter
|
108
108
|
end
|
109
|
+
|
110
|
+
def self.arel_table_from_key_values(partition_key_values, as = nil)
|
111
|
+
@arel_tables ||= {}
|
112
|
+
new_arel_table = @arel_tables[[partition_key_values, as]]
|
113
|
+
|
114
|
+
unless new_arel_table
|
115
|
+
arel_engine_hash = {:engine => self.arel_engine, :as => as}
|
116
|
+
new_arel_table = Arel::Table.new(self.partition_table_name(*partition_key_values), arel_engine_hash)
|
117
|
+
@arel_tables[[partition_key_values, as]] = new_arel_table
|
118
|
+
end
|
109
119
|
|
120
|
+
return new_arel_table
|
121
|
+
end
|
122
|
+
|
110
123
|
#
|
111
124
|
# In activerecord 3.0 we need to supply an Arel::Table for the key value(s) used
|
112
125
|
# to determine the specific child table to access.
|
@@ -115,13 +128,8 @@ module Partitioned
|
|
115
128
|
# @param [String] as (nil) the name of the table associated with this Arel::Table
|
116
129
|
# @return [Arel::Table] the generated Arel::Table
|
117
130
|
def self.dynamic_arel_table(values, as = nil)
|
118
|
-
@arel_tables ||= {}
|
119
131
|
key_values = self.partition_key_values(values)
|
120
|
-
|
121
|
-
arel_engine_hash = {:engine => self.arel_engine}
|
122
|
-
arel_engine_hash[:as] = as unless as.blank?
|
123
|
-
new_arel_table = Arel::Table.new(self.partition_table_name(*key_values), arel_engine_hash)
|
124
|
-
return new_arel_table
|
132
|
+
return arel_table_from_key_values(key_values, as)
|
125
133
|
end
|
126
134
|
|
127
135
|
#
|
@@ -131,9 +139,8 @@ module Partitioned
|
|
131
139
|
# @param [String] as (nil) the name of the table associated with the Arel::Table
|
132
140
|
# @return [Arel::Table] the generated Arel::Table
|
133
141
|
def dynamic_arel_table(as = nil)
|
134
|
-
|
135
|
-
|
136
|
-
return self.class.dynamic_arel_table(key_values, as)
|
142
|
+
key_values = self.class.partition_key_values(attributes)
|
143
|
+
return self.class.arel_table_from_key_values(key_values, as)
|
137
144
|
end
|
138
145
|
|
139
146
|
#
|
@@ -149,10 +156,9 @@ module Partitioned
|
|
149
156
|
#
|
150
157
|
# @param [*Array<Object>] partition_field the field values to partition on
|
151
158
|
# @return [Hash] the scoping
|
152
|
-
def self.from_partition(*
|
153
|
-
table_alias_name = partition_table_alias_name(*
|
154
|
-
|
155
|
-
tap{|relation| relation.table.table_alias = table_alias_name}
|
159
|
+
def self.from_partition(*partition_key_values)
|
160
|
+
table_alias_name = partition_table_alias_name(*partition_key_values)
|
161
|
+
return ActiveRecord::Relation.new(self, self.arel_table_from_key_values(partition_key_values, table_alias_name))
|
156
162
|
end
|
157
163
|
|
158
164
|
#
|
@@ -177,10 +183,8 @@ module Partitioned
|
|
177
183
|
#
|
178
184
|
# @param [*Array<Object>] partition_field the field values to partition on
|
179
185
|
# @return [Hash] the scoping
|
180
|
-
def self.from_partition_without_alias(*
|
181
|
-
|
182
|
-
from(table_alias_name).
|
183
|
-
tap{|relation| relation.table.table_alias = table_alias_name}
|
186
|
+
def self.from_partition_without_alias(*partition_key_values)
|
187
|
+
return ActiveRecord::Relation.new(self, self.arel_table_from_key_values(partition_key_values, nil))
|
184
188
|
end
|
185
189
|
|
186
190
|
#
|
@@ -305,7 +309,7 @@ module Partitioned
|
|
305
309
|
# A reasonable alias for this table
|
306
310
|
#
|
307
311
|
partition.table_alias_name lambda {|model, *partition_key_values|
|
308
|
-
return model.
|
312
|
+
return model.table_name
|
309
313
|
}
|
310
314
|
|
311
315
|
#
|
data/lib/partitioned/version.rb
CHANGED
data/lib/partitioned.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'monkey_patch_activerecord'
|
2
2
|
require 'monkey_patch_postgres'
|
3
|
+
require 'monkey_patch_redshift'
|
3
4
|
|
4
5
|
require 'partitioned/active_record_overrides'
|
5
6
|
require 'partitioned/partitioned_base/configurator.rb'
|
@@ -9,6 +10,7 @@ require 'partitioned/partitioned_base.rb'
|
|
9
10
|
require 'partitioned/partitioned_base/configurator/reader'
|
10
11
|
require 'partitioned/partitioned_base/partition_manager'
|
11
12
|
require 'partitioned/partitioned_base/sql_adapter'
|
13
|
+
require 'partitioned/partitioned_base/redshift_sql_adapter'
|
12
14
|
|
13
15
|
require 'partitioned/by_time_field'
|
14
16
|
require 'partitioned/by_yearly_time_field'
|
data/partitioned.gemspec
CHANGED
@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
|
|
7
7
|
s.name = 'partitioned'
|
8
8
|
s.version = Partitioned::VERSION
|
9
9
|
s.license = 'New BSD License'
|
10
|
-
s.date = '2013-
|
10
|
+
s.date = '2013-02-04'
|
11
11
|
s.summary = "Postgres table partitioning support for ActiveRecord."
|
12
12
|
s.description = "A gem providing support for table partitioning in ActiveRecord. Support is only available for postgres databases. Other features include child table management (creation and deletion) and bulk data creating and updating."
|
13
13
|
s.authors = ["Keith Gabryelski", "Aleksandr Dembskiy"]
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: partitioned
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.5
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2013-
|
13
|
+
date: 2013-02-04 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: pg
|
@@ -106,6 +106,7 @@ files:
|
|
106
106
|
- init.rb
|
107
107
|
- lib/monkey_patch_activerecord.rb
|
108
108
|
- lib/monkey_patch_postgres.rb
|
109
|
+
- lib/monkey_patch_redshift.rb
|
109
110
|
- lib/partitioned.rb
|
110
111
|
- lib/partitioned/active_record_overrides.rb
|
111
112
|
- lib/partitioned/by_created_at.rb
|
@@ -127,6 +128,7 @@ files:
|
|
127
128
|
- lib/partitioned/partitioned_base/configurator/dsl.rb
|
128
129
|
- lib/partitioned/partitioned_base/configurator/reader.rb
|
129
130
|
- lib/partitioned/partitioned_base/partition_manager.rb
|
131
|
+
- lib/partitioned/partitioned_base/redshift_sql_adapter.rb
|
130
132
|
- lib/partitioned/partitioned_base/sql_adapter.rb
|
131
133
|
- lib/partitioned/version.rb
|
132
134
|
- lib/tasks/desirable_tasks.rake
|
@@ -201,7 +203,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
201
203
|
version: '0'
|
202
204
|
requirements: []
|
203
205
|
rubyforge_project:
|
204
|
-
rubygems_version: 1.8.
|
206
|
+
rubygems_version: 1.8.25
|
205
207
|
signing_key:
|
206
208
|
specification_version: 3
|
207
209
|
summary: Postgres table partitioning support for ActiveRecord.
|