partitioned 1.1.10 → 1.1.11
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +2 -0
- data/Gemfile +1 -15
- data/{README → README.md} +111 -61
- data/lib/partitioned/partitioned_base/partition_manager.rb +10 -2
- data/lib/partitioned/partitioned_base/sql_adapter.rb +21 -0
- data/lib/partitioned/version.rb +1 -1
- data/partitioned.gemspec +20 -17
- metadata +21 -5
- data/Gemfile.lock +0 -120
data/.gitignore
ADDED
data/Gemfile
CHANGED
@@ -1,17 +1,3 @@
|
|
1
1
|
source "http://rubygems.org"
|
2
2
|
|
3
|
-
|
4
|
-
# Bundler will treat runtime dependencies like base dependencies, and
|
5
|
-
# development dependencies will be added by default to the :development group.
|
6
|
-
gemspec
|
7
|
-
|
8
|
-
# jquery-rails is used by the dummy application
|
9
|
-
gem "jquery-rails"
|
10
|
-
|
11
|
-
# Declare any dependencies that are still in development here instead of in
|
12
|
-
# your gemspec. These might include edge Rails or gems from your path or
|
13
|
-
# Git. Remember to move these dependencies to your gemspec before releasing
|
14
|
-
# your gem to rubygems.org.
|
15
|
-
|
16
|
-
# To use debugger
|
17
|
-
# gem 'ruby-debug'
|
3
|
+
gemspec
|
data/{README → README.md}
RENAMED
@@ -1,5 +1,4 @@
|
|
1
|
-
Partitioned
|
2
|
-
===========
|
1
|
+
# Partitioned
|
3
2
|
|
4
3
|
Partitioned adds assistance to ActiveRecord for manipulating (reading,
|
5
4
|
creating, updating) an activerecord model that represents data that
|
@@ -26,83 +25,109 @@ provides streamlined targeted access to the desired data.
|
|
26
25
|
Support for bulk inserts and bulk updates is also provided via
|
27
26
|
Partitioned::Base.create_many and Partitioned::Base.update_many.
|
28
27
|
|
29
|
-
Example
|
30
|
-
=======
|
28
|
+
## Example
|
31
29
|
|
32
|
-
Given the following models:
|
33
30
|
|
34
|
-
|
35
|
-
end
|
31
|
+
Given the following models:
|
36
32
|
|
37
|
-
|
38
|
-
|
33
|
+
```ruby
|
34
|
+
class Company < ActiveRecord::Base
|
35
|
+
end
|
39
36
|
|
40
|
-
|
37
|
+
class ByCompanyId < Partitioned::ByForeignKey
|
38
|
+
self.abstract_class = true
|
41
39
|
|
42
|
-
|
43
|
-
return :company_id
|
44
|
-
end
|
40
|
+
belongs_to :company
|
45
41
|
|
46
|
-
|
47
|
-
|
48
|
-
end
|
42
|
+
def self.partition_foreign_key
|
43
|
+
return :company_id
|
49
44
|
end
|
50
45
|
|
51
|
-
|
46
|
+
partitioned do |partition|
|
47
|
+
partition.index :id, :unique => true
|
52
48
|
end
|
49
|
+
end
|
50
|
+
|
51
|
+
class Employee < ByCompanyId
|
52
|
+
end
|
53
|
+
```
|
53
54
|
|
54
55
|
and the following tables:
|
55
56
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
57
|
+
```sql
|
58
|
+
-- this is the referenced table
|
59
|
+
create table companies
|
60
|
+
(
|
61
|
+
id serial not null primary key,
|
62
|
+
created_at timestamp not null default now(),
|
63
|
+
updated_at timestamp,
|
64
|
+
name text null
|
65
|
+
);
|
66
|
+
|
67
|
+
-- add some companies
|
68
|
+
insert into table companies (name) values
|
69
|
+
('company 1'),('company 2'),('company 2');
|
70
|
+
|
71
|
+
-- this is the parent table
|
72
|
+
create table employees
|
73
|
+
(
|
74
|
+
id serial not null primary key,
|
75
|
+
created_at timestamp not null default now(),
|
76
|
+
updated_at timestamp,
|
77
|
+
name text null,
|
78
|
+
company_id integer not null references companies
|
79
|
+
);
|
80
|
+
```
|
78
81
|
|
79
82
|
We now need to create some infrastructure for partitioned tables,
|
80
83
|
in particular, we create a schema to hold the child partition
|
81
84
|
tables of employees.
|
82
85
|
|
83
|
-
|
86
|
+
```ruby
|
87
|
+
Employee.create_infrastructure
|
88
|
+
```
|
84
89
|
|
85
90
|
Which creates the employees_partitions schema using the following SQL:
|
86
91
|
|
87
|
-
|
92
|
+
```sql
|
93
|
+
create schema employees_partitions;
|
94
|
+
```
|
88
95
|
|
89
96
|
NOTE: We also install protections on the employees table so it isn't
|
90
97
|
used as a data table (this SQL is not presented for simplicity but is
|
91
98
|
apart of the create_infrastructure call).
|
92
99
|
|
100
|
+
You can create migration for this Employee in this case:
|
101
|
+
|
102
|
+
```ruby
|
103
|
+
class CreatePartitionEmployee < ActiveRecord::Migration
|
104
|
+
def up
|
105
|
+
Employee.create_infrastructure
|
106
|
+
end
|
107
|
+
|
108
|
+
def down
|
109
|
+
Employee.delete_infrastructure
|
110
|
+
end
|
111
|
+
end
|
112
|
+
```
|
113
|
+
|
93
114
|
To add child tables we use the create_new_partitions_tables method:
|
94
115
|
|
95
|
-
|
96
|
-
|
116
|
+
```ruby
|
117
|
+
company_ids = Company.all.map(&:id)
|
118
|
+
Employee.create_new_partition_tables(company_ids)
|
119
|
+
```
|
97
120
|
|
98
121
|
which results in the following SQL:
|
99
122
|
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
123
|
+
```sql
|
124
|
+
create table employees_partitions.p1
|
125
|
+
( CHECK ( company_id = 1 ) ) INHERITS (employees);
|
126
|
+
create table employees_partitions.p2
|
127
|
+
( CHECK ( company_id = 2 ) ) INHERITS (employees);
|
128
|
+
create table employees_partitions.p3
|
129
|
+
( CHECK ( company_id = 3 ) ) INHERITS (employees);
|
130
|
+
```
|
106
131
|
|
107
132
|
NOTE: Some other SQL is generated in the above example, specifically
|
108
133
|
the reference to the companies table needs to be explicitly created
|
@@ -117,13 +142,22 @@ inserts of the EMPLOYEES table redirect the record insert into the
|
|
117
142
|
specific child table determined by the value of COMPANY_ID
|
118
143
|
|
119
144
|
eg:
|
120
|
-
|
145
|
+
|
146
|
+
```ruby
|
147
|
+
employee = Employee.create(:name => 'Keith', :company_id => 1)
|
148
|
+
```
|
121
149
|
|
122
150
|
this would normally produce the following:
|
123
|
-
|
151
|
+
|
152
|
+
```sql
|
153
|
+
INSERT INTO employees ('name', company_id) values ('Keith', 1);
|
154
|
+
```
|
124
155
|
|
125
156
|
but with Partitioned we see:
|
126
|
-
|
157
|
+
|
158
|
+
```sql
|
159
|
+
INSERT INTO employees_partitions.p1 ('name', company_id) values ('Keith', 1);
|
160
|
+
```
|
127
161
|
|
128
162
|
reads of such a table need some assistance to find the specific child
|
129
163
|
table the record exists in.
|
@@ -134,15 +168,19 @@ search all child table for the specific record we are looking for.
|
|
134
168
|
|
135
169
|
This is no longer valid (well, doesn't perform well):
|
136
170
|
|
137
|
-
|
171
|
+
```ruby
|
172
|
+
employee = Employee.find(1)
|
173
|
+
```
|
138
174
|
|
139
175
|
instead, do one of the following:
|
140
176
|
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
177
|
+
```ruby
|
178
|
+
employee = Employee.from_partition(1).find(1)
|
179
|
+
employee = Employee.find(:first,
|
180
|
+
:conditions => {:name => 'Keith', :company_id => 1})
|
181
|
+
employee = Employee.find(:first,
|
182
|
+
:conditions => {:id => 1, :company_id => 1})
|
183
|
+
```
|
146
184
|
|
147
185
|
an update (employee.save where the record already exists in the
|
148
186
|
database) will take advantage of knowing which child table the record
|
@@ -150,11 +188,23 @@ exists in so it can do some optimization.
|
|
150
188
|
|
151
189
|
so, the following works as expected:
|
152
190
|
|
153
|
-
|
154
|
-
|
191
|
+
```ruby
|
192
|
+
employee.name = "Not Keith"
|
193
|
+
employee.save
|
194
|
+
```
|
155
195
|
|
156
196
|
turns into the following SQL:
|
157
197
|
|
158
|
-
|
198
|
+
```sql
|
199
|
+
update employees_partitions.p1 set name = 'Not Keith' where id = 1;
|
200
|
+
```
|
201
|
+
|
202
|
+
## Contributing
|
203
|
+
|
204
|
+
1. Fork it
|
205
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
206
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
207
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
208
|
+
5. Create new Pull Request
|
159
209
|
|
160
210
|
Copyright 2010-2012 fiksu.com, inc, all rights reserved
|
@@ -67,6 +67,14 @@ module Partitioned
|
|
67
67
|
add_parent_table_rules
|
68
68
|
end
|
69
69
|
|
70
|
+
#
|
71
|
+
# The once called function to drop a parent the schema.
|
72
|
+
#
|
73
|
+
def delete_infrastructure
|
74
|
+
drop_partition_schema
|
75
|
+
remove_parent_table_rules
|
76
|
+
end
|
77
|
+
|
70
78
|
#
|
71
79
|
# An array of key values (each key value is an array of keys) that represent
|
72
80
|
# the child partitions that should be created.
|
@@ -173,8 +181,8 @@ module Partitioned
|
|
173
181
|
extend Forwardable
|
174
182
|
def_delegators :parent_table_class, :sql_adapter, :configurator
|
175
183
|
def_delegators :sql_adapter, :drop_partition_table, :create_partition_table, :add_partition_table_index,
|
176
|
-
:add_references_to_partition_table, :create_partition_schema, :add_parent_table_rules,
|
177
|
-
:partition_table_name, :partition_table_alias_name
|
184
|
+
:add_references_to_partition_table, :create_partition_schema, :drop_partition_schema, :add_parent_table_rules,
|
185
|
+
:remove_parent_table_rules, :partition_table_name, :partition_table_alias_name
|
178
186
|
|
179
187
|
end
|
180
188
|
end
|
@@ -44,6 +44,15 @@ module Partitioned
|
|
44
44
|
create_schema(configurator.schema_name, :unless_exists => true)
|
45
45
|
end
|
46
46
|
|
47
|
+
#
|
48
|
+
# Child tables whose parent table is 'foos', typically exist in a schema named foos_partitions.
|
49
|
+
#
|
50
|
+
# *partition_key_values are needed here to support the use of multiple schemas to keep tables in.
|
51
|
+
#
|
52
|
+
def drop_partition_schema(*partition_key_values)
|
53
|
+
drop_schema(configurator.schema_name, :cascade => true)
|
54
|
+
end
|
55
|
+
|
47
56
|
#
|
48
57
|
# Does a specific child partition exist.
|
49
58
|
#
|
@@ -119,6 +128,18 @@ module Partitioned
|
|
119
128
|
execute(sql)
|
120
129
|
end
|
121
130
|
|
131
|
+
#
|
132
|
+
# Used to drop the parent table rule.
|
133
|
+
#
|
134
|
+
def remove_parent_table_rules(*partition_key_values)
|
135
|
+
insert_redirector_name = parent_table_rule_name("insert", "redirector", *partition_key_values)
|
136
|
+
sql = <<-SQL
|
137
|
+
DROP RULE #{insert_redirector_name}
|
138
|
+
ON INSERT CASCADE
|
139
|
+
SQL
|
140
|
+
execute(sql)
|
141
|
+
end
|
142
|
+
|
122
143
|
#
|
123
144
|
# The name of the table (schemaname.childtablename) given the check constraint values.
|
124
145
|
#
|
data/lib/partitioned/version.rb
CHANGED
data/partitioned.gemspec
CHANGED
@@ -4,21 +4,24 @@ $LOAD_PATH.push File.expand_path("../lib", __FILE__)
|
|
4
4
|
require "partitioned/version"
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
7
|
+
s.name = 'partitioned'
|
8
|
+
s.version = Partitioned::VERSION
|
9
|
+
s.license = 'New BSD License'
|
10
|
+
s.date = '2013-04-29'
|
11
|
+
s.summary = "Postgres table partitioning support for ActiveRecord."
|
12
|
+
s.description = "A gem providing support for table partitioning in ActiveRecord. Support is available for postgres and AWS RedShift databases. Other features include child table management (creation and deletion) and bulk data creating and updating."
|
13
|
+
s.authors = ["Keith Gabryelski", "Aleksandr Dembskiy", "Edward Slavich"]
|
14
|
+
s.email = 'keith@fiksu.com'
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
s.require_path = 'lib'
|
18
|
+
s.homepage = 'http://github.com/fiksu/partitioned'
|
19
|
+
|
20
|
+
s.add_development_dependency "jquery-rails"
|
21
|
+
|
22
|
+
s.add_dependency 'pg'
|
23
|
+
s.add_dependency 'rails', '>= 3.2.8'
|
24
|
+
s.add_dependency 'rspec-rails'
|
25
|
+
s.add_dependency 'bulk_data_methods', '1.0.0'
|
26
|
+
s.add_dependency 'activerecord-redshift-adapter'
|
24
27
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: partitioned
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.11
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -11,8 +11,24 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date: 2013-04-
|
14
|
+
date: 2013-04-29 00:00:00.000000000 Z
|
15
15
|
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
17
|
+
name: jquery-rails
|
18
|
+
requirement: !ruby/object:Gem::Requirement
|
19
|
+
none: false
|
20
|
+
requirements:
|
21
|
+
- - ! '>='
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: '0'
|
24
|
+
type: :development
|
25
|
+
prerelease: false
|
26
|
+
version_requirements: !ruby/object:Gem::Requirement
|
27
|
+
none: false
|
28
|
+
requirements:
|
29
|
+
- - ! '>='
|
30
|
+
- !ruby/object:Gem::Version
|
31
|
+
version: '0'
|
16
32
|
- !ruby/object:Gem::Dependency
|
17
33
|
name: pg
|
18
34
|
requirement: !ruby/object:Gem::Requirement
|
@@ -101,12 +117,12 @@ executables: []
|
|
101
117
|
extensions: []
|
102
118
|
extra_rdoc_files: []
|
103
119
|
files:
|
120
|
+
- .gitignore
|
104
121
|
- .rspec
|
105
122
|
- Gemfile
|
106
|
-
- Gemfile.lock
|
107
123
|
- LICENSE
|
108
124
|
- PARTITIONING_EXPLAINED.txt
|
109
|
-
- README
|
125
|
+
- README.md
|
110
126
|
- Rakefile
|
111
127
|
- examples/README
|
112
128
|
- examples/company_id.rb
|
@@ -220,7 +236,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
220
236
|
version: '0'
|
221
237
|
requirements: []
|
222
238
|
rubyforge_project:
|
223
|
-
rubygems_version: 1.8.
|
239
|
+
rubygems_version: 1.8.24
|
224
240
|
signing_key:
|
225
241
|
specification_version: 3
|
226
242
|
summary: Postgres table partitioning support for ActiveRecord.
|
data/Gemfile.lock
DELETED
@@ -1,120 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
partitioned (1.1.9)
|
5
|
-
activerecord-redshift-adapter
|
6
|
-
bulk_data_methods (= 1.0.0)
|
7
|
-
pg
|
8
|
-
rails (>= 3.2.8)
|
9
|
-
rspec-rails
|
10
|
-
|
11
|
-
GEM
|
12
|
-
remote: http://rubygems.org/
|
13
|
-
specs:
|
14
|
-
actionmailer (3.2.13)
|
15
|
-
actionpack (= 3.2.13)
|
16
|
-
mail (~> 2.5.3)
|
17
|
-
actionpack (3.2.13)
|
18
|
-
activemodel (= 3.2.13)
|
19
|
-
activesupport (= 3.2.13)
|
20
|
-
builder (~> 3.0.0)
|
21
|
-
erubis (~> 2.7.0)
|
22
|
-
journey (~> 1.0.4)
|
23
|
-
rack (~> 1.4.5)
|
24
|
-
rack-cache (~> 1.2)
|
25
|
-
rack-test (~> 0.6.1)
|
26
|
-
sprockets (~> 2.2.1)
|
27
|
-
activemodel (3.2.13)
|
28
|
-
activesupport (= 3.2.13)
|
29
|
-
builder (~> 3.0.0)
|
30
|
-
activerecord (3.2.13)
|
31
|
-
activemodel (= 3.2.13)
|
32
|
-
activesupport (= 3.2.13)
|
33
|
-
arel (~> 3.0.2)
|
34
|
-
tzinfo (~> 0.3.29)
|
35
|
-
activerecord-redshift-adapter (0.9.0)
|
36
|
-
pg
|
37
|
-
rails (>= 3.0.0)
|
38
|
-
activeresource (3.2.13)
|
39
|
-
activemodel (= 3.2.13)
|
40
|
-
activesupport (= 3.2.13)
|
41
|
-
activesupport (3.2.13)
|
42
|
-
i18n (= 0.6.1)
|
43
|
-
multi_json (~> 1.0)
|
44
|
-
arel (3.0.2)
|
45
|
-
builder (3.0.4)
|
46
|
-
bulk_data_methods (1.0.0)
|
47
|
-
pg
|
48
|
-
rails (>= 3.0.0)
|
49
|
-
rspec-rails
|
50
|
-
diff-lcs (1.2.4)
|
51
|
-
erubis (2.7.0)
|
52
|
-
hike (1.2.2)
|
53
|
-
i18n (0.6.1)
|
54
|
-
journey (1.0.4)
|
55
|
-
jquery-rails (2.2.1)
|
56
|
-
railties (>= 3.0, < 5.0)
|
57
|
-
thor (>= 0.14, < 2.0)
|
58
|
-
json (1.7.7)
|
59
|
-
mail (2.5.3)
|
60
|
-
i18n (>= 0.4.0)
|
61
|
-
mime-types (~> 1.16)
|
62
|
-
treetop (~> 1.4.8)
|
63
|
-
mime-types (1.23)
|
64
|
-
multi_json (1.7.2)
|
65
|
-
pg (0.15.1)
|
66
|
-
polyglot (0.3.3)
|
67
|
-
rack (1.4.5)
|
68
|
-
rack-cache (1.2)
|
69
|
-
rack (>= 0.4)
|
70
|
-
rack-ssl (1.3.3)
|
71
|
-
rack
|
72
|
-
rack-test (0.6.2)
|
73
|
-
rack (>= 1.0)
|
74
|
-
rails (3.2.13)
|
75
|
-
actionmailer (= 3.2.13)
|
76
|
-
actionpack (= 3.2.13)
|
77
|
-
activerecord (= 3.2.13)
|
78
|
-
activeresource (= 3.2.13)
|
79
|
-
activesupport (= 3.2.13)
|
80
|
-
bundler (~> 1.0)
|
81
|
-
railties (= 3.2.13)
|
82
|
-
railties (3.2.13)
|
83
|
-
actionpack (= 3.2.13)
|
84
|
-
activesupport (= 3.2.13)
|
85
|
-
rack-ssl (~> 1.3.2)
|
86
|
-
rake (>= 0.8.7)
|
87
|
-
rdoc (~> 3.4)
|
88
|
-
thor (>= 0.14.6, < 2.0)
|
89
|
-
rake (10.0.4)
|
90
|
-
rdoc (3.12.2)
|
91
|
-
json (~> 1.4)
|
92
|
-
rspec-core (2.13.1)
|
93
|
-
rspec-expectations (2.13.0)
|
94
|
-
diff-lcs (>= 1.1.3, < 2.0)
|
95
|
-
rspec-mocks (2.13.1)
|
96
|
-
rspec-rails (2.13.0)
|
97
|
-
actionpack (>= 3.0)
|
98
|
-
activesupport (>= 3.0)
|
99
|
-
railties (>= 3.0)
|
100
|
-
rspec-core (~> 2.13.0)
|
101
|
-
rspec-expectations (~> 2.13.0)
|
102
|
-
rspec-mocks (~> 2.13.0)
|
103
|
-
sprockets (2.2.2)
|
104
|
-
hike (~> 1.2)
|
105
|
-
multi_json (~> 1.0)
|
106
|
-
rack (~> 1.0)
|
107
|
-
tilt (~> 1.1, != 1.3.0)
|
108
|
-
thor (0.18.1)
|
109
|
-
tilt (1.3.7)
|
110
|
-
treetop (1.4.12)
|
111
|
-
polyglot
|
112
|
-
polyglot (>= 0.3.1)
|
113
|
-
tzinfo (0.3.37)
|
114
|
-
|
115
|
-
PLATFORMS
|
116
|
-
ruby
|
117
|
-
|
118
|
-
DEPENDENCIES
|
119
|
-
jquery-rails
|
120
|
-
partitioned!
|