pg_shrink 0.0.7 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -2
- data/Shrinkfile.example +10 -5
- data/lib/pg_shrink/database/postgres.rb +15 -13
- data/lib/pg_shrink/sub_table_filter.rb +11 -28
- data/lib/pg_shrink/sub_table_operator.rb +5 -11
- data/lib/pg_shrink/sub_table_sanitizer.rb +4 -3
- data/lib/pg_shrink/version.rb +1 -1
- data/pg_shrink.gemspec +3 -3
- data/spec/pg_shrink/database/postgres_spec.rb +115 -28
- data/spec/pg_shrink/table_spec.rb +3 -5
- data/spec/pg_shrink_spec.rb +68 -71
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 79d28599347bf3d3b21a5ff1066eb005d93de451
|
4
|
+
data.tar.gz: 52afef4d72dfe607d9a6b9fea197789492a27153
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b34ea492d943779dafde384932065fdaae0d5244ff09b115c5436a319c27bdccae5ac01c02dcfda578a54a3383b725f561c5333fc199efd8e91dae81ad542303
|
7
|
+
data.tar.gz: b3c7736462f971e4eba50960a48dea9260c891d1b0152c7fe675669213ac241e459a9dce38f6a32cdfb8e5765d17a898e78ed756aa95abe4476cb233f8aad103
|
data/README.md
CHANGED
@@ -1,5 +1,4 @@
|
|
1
|
-
[![Build Status](https://travis-ci.org/apartmentlist/
|
2
|
-
|
1
|
+
[![Build Status](https://travis-ci.org/apartmentlist/pg_shrink.svg?branch=master)](https://travis-ci.org/apartmentlist/pg_shrink)
|
3
2
|
# PgShrink
|
4
3
|
|
5
4
|
The pg_shrink tool makes it easy to shrink and sanitize a postgres database,
|
data/Shrinkfile.example
CHANGED
@@ -23,11 +23,16 @@ filter_table :users do |f|
|
|
23
23
|
f.filter_subtable(:email_preferences, :foreign_key => :user_email,
|
24
24
|
:primary_key => :email)
|
25
25
|
|
26
|
-
# You can also
|
27
|
-
#
|
28
|
-
|
29
|
-
|
30
|
-
|
26
|
+
# You can also pass an additional where clause expression that restricts
|
27
|
+
# filtering propagation to certain subset of rows. This allows to support
|
28
|
+
# polymorphic relationships, or STI.
|
29
|
+
f.filter_subtable(:email_preferences, foreign_key: user_id,
|
30
|
+
where: "type = 'User::Registered'")
|
31
|
+
|
32
|
+
# 'where' clause can be anything that sequel supports
|
33
|
+
f.filter_subtable(:email_preferences, foreign_key: user_id,
|
34
|
+
where: { type: 'User::Registered',
|
35
|
+
color: 'Red'})
|
31
36
|
|
32
37
|
# If it feels more natural, you can define additional filters
|
33
38
|
# within a filter_subtable definitition
|
@@ -113,31 +113,33 @@ module PgShrink
|
|
113
113
|
end
|
114
114
|
|
115
115
|
def propagate_delete(opts)
|
116
|
+
|
117
|
+
child_table = opts[:child_table]
|
118
|
+
parent_table = opts[:parent_table]
|
119
|
+
child_key = opts[:child_key]
|
120
|
+
parent_key = opts[:parent_key]
|
121
|
+
where_clause = opts[:where]
|
122
|
+
|
116
123
|
# what we conceptually want to do is delete the left outer join where id is null.
|
117
124
|
# That's not working in postgres, so we instead use where not exists. Docs
|
118
125
|
# indicate using where not exists and select 1 in this case.
|
119
126
|
# See:
|
120
127
|
# http://www.postgresql.org/docs/current/interactive/functions-subquery.html#FUNCTIONS-SUBQUERY-EXISTS
|
121
|
-
query = "DELETE FROM #{
|
122
|
-
"SELECT 1 from #{
|
123
|
-
"#{
|
124
|
-
"#{
|
128
|
+
query = "DELETE FROM #{child_table} WHERE NOT EXISTS (" +
|
129
|
+
"SELECT 1 from #{parent_table} where " +
|
130
|
+
"#{child_table}.#{child_key} = " +
|
131
|
+
"#{parent_table}.#{parent_key}" +
|
125
132
|
")"
|
126
133
|
|
134
|
+
query_builder = connection.from(child_table)
|
135
|
+
query_builder = query_builder.where(where_clause) if where_clause
|
127
136
|
|
128
137
|
# Outside of the join statements, we want to maintain the ease of hash-based
|
129
138
|
# conditions. Do this by using a query builder but then swapping in delete SQL
|
130
139
|
# in the end.
|
131
|
-
|
132
|
-
Array.wrap(opts[:conditions]).compact.each do |cond|
|
133
|
-
query_builder = query_builder.where(cond)
|
134
|
-
end
|
135
|
-
Array.wrap(opts[:exclude]).compact.each do |exclude_cond|
|
136
|
-
query_builder = query_builder.exclude(exclude_cond)
|
137
|
-
end
|
140
|
+
|
138
141
|
sql = query_builder.sql.gsub("WHERE", "AND").
|
139
|
-
gsub("SELECT * FROM \"#{
|
140
|
-
query)
|
142
|
+
gsub("SELECT * FROM \"#{child_table}\"", query)
|
141
143
|
|
142
144
|
connection[sql].delete
|
143
145
|
end
|
@@ -4,42 +4,25 @@ module PgShrink
|
|
4
4
|
def propagate_table!
|
5
5
|
primary_key = @opts[:primary_key]
|
6
6
|
foreign_key = @opts[:foreign_key]
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
end
|
11
|
-
self.database.log("Beginning subtable propagation from " +
|
7
|
+
where_clause = @opts[:where]
|
8
|
+
|
9
|
+
self.database.log('Beginning subtable propagation from ' +
|
12
10
|
"#{self.parent.table_name} to #{self.table.table_name}")
|
13
|
-
self.database.propagate_delete(:parent_table => self.parent.table_name,
|
14
|
-
:child_table => self.table.table_name,
|
15
|
-
:parent_key => primary_key,
|
16
|
-
:child_key => foreign_key,
|
17
|
-
:conditions => additional_conditions)
|
18
11
|
|
19
|
-
self.database.
|
12
|
+
self.database.propagate_delete(parent_table: self.parent.table_name,
|
13
|
+
child_table: self.table.table_name,
|
14
|
+
parent_key: primary_key,
|
15
|
+
child_key: foreign_key,
|
16
|
+
where: where_clause)
|
17
|
+
|
18
|
+
self.database.log('Done with subtable propagation from ' +
|
20
19
|
"#{self.parent.table_name} to #{self.table.table_name}")
|
20
|
+
|
21
21
|
if self.table.subtable_filters.any?
|
22
22
|
self.database.vacuum_and_reindex!(self.table.table_name)
|
23
23
|
self.table.subtable_filters.each(&:propagate_table!)
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
27
|
-
def propagate!(old_parent_data, new_parent_data)
|
28
|
-
return if (old_parent_data.empty? && new_parent_data.empty?)
|
29
|
-
old_batch_keys = old_parent_data.map {|record| record[@opts[:primary_key]]}
|
30
|
-
new_batch_keys = new_parent_data.map {|record| record[@opts[:primary_key]]}
|
31
|
-
|
32
|
-
foreign_key = @opts[:foreign_key]
|
33
|
-
finder_options = {foreign_key => old_batch_keys}
|
34
|
-
if @opts[:type_key] && @opts[:type]
|
35
|
-
finder_options[@opts[:type_key]] = @opts[:type]
|
36
|
-
end
|
37
|
-
|
38
|
-
old_records = table.get_records(finder_options)
|
39
|
-
table.filter_batch(old_records) do |record|
|
40
|
-
new_batch_keys.include?(record[foreign_key])
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
27
|
end
|
45
28
|
end
|
@@ -2,9 +2,10 @@ module PgShrink
|
|
2
2
|
class SubTableOperator
|
3
3
|
attr_accessor :parent, :table_name, :database
|
4
4
|
def default_opts
|
5
|
-
{
|
5
|
+
{
|
6
|
+
:foreign_key =>
|
6
7
|
"#{ActiveSupport::Inflector.singularize(parent.table_name.to_s)}_id",
|
7
|
-
|
8
|
+
:primary_key => :id
|
8
9
|
}
|
9
10
|
end
|
10
11
|
|
@@ -16,15 +17,6 @@ module PgShrink
|
|
16
17
|
database.table(table_name)
|
17
18
|
end
|
18
19
|
|
19
|
-
def validate_opts!(opts)
|
20
|
-
if opts[:type_key] && !opts[:type]
|
21
|
-
raise "Error: #{name} has type_key set but no type"
|
22
|
-
end
|
23
|
-
if opts[:type] && !opts[:type_key]
|
24
|
-
raise "Error: #{name} has type set but no type_key"
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
20
|
def initialize(parent, table_name, opts = {})
|
29
21
|
self.parent = parent
|
30
22
|
self.table_name = table_name
|
@@ -34,6 +26,8 @@ module PgShrink
|
|
34
26
|
validate_opts!(@opts)
|
35
27
|
end
|
36
28
|
|
29
|
+
def validate_opts!(opts); end
|
30
|
+
|
37
31
|
def propagate!(old_parent_data, new_parent_data)
|
38
32
|
raise "Implement in subclass"
|
39
33
|
end
|
@@ -15,9 +15,10 @@ module PgShrink
|
|
15
15
|
|
16
16
|
foreign_key = @opts[:foreign_key]
|
17
17
|
finder_options = {foreign_key => old_batch.keys}
|
18
|
-
|
19
|
-
|
20
|
-
|
18
|
+
|
19
|
+
# adding additional filters from where clause.
|
20
|
+
# TODO: support where clauses using non hash syntax (string, symbol)
|
21
|
+
finder_options.merge!(@opts[:where])
|
21
22
|
|
22
23
|
parent_field = @opts[:local_field].to_sym
|
23
24
|
child_field = @opts[:foreign_field].to_sym
|
data/lib/pg_shrink/version.rb
CHANGED
data/pg_shrink.gemspec
CHANGED
@@ -6,11 +6,11 @@ require 'pg_shrink/version'
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = 'pg_shrink'
|
8
8
|
spec.version = PgShrink::VERSION
|
9
|
-
spec.authors = ['Kevin Ball']
|
10
|
-
spec.email = ['kmball11@gmail.com']
|
9
|
+
spec.authors = ['Kevin Ball', 'Matt Nemenman']
|
10
|
+
spec.email = ['kmball11@gmail.com', 'matt@apartmentlist.com']
|
11
11
|
spec.description = 'pg_shrink makes it simple to shrink and sanitize a PosrgreSQL database'
|
12
12
|
spec.summary = 'pg_shrink'
|
13
|
-
spec.homepage = 'https://github.com/apartmentlist/
|
13
|
+
spec.homepage = 'https://github.com/apartmentlist/pg_shrink'
|
14
14
|
spec.license = 'MIT'
|
15
15
|
|
16
16
|
spec.files = `git ls-files`.split($/)
|
@@ -8,29 +8,36 @@ describe PgShrink::Database::Postgres do
|
|
8
8
|
|
9
9
|
before(:all) do
|
10
10
|
PgSpecHelper.reset_database
|
11
|
+
|
12
|
+
# id column created implicitly
|
11
13
|
PgSpecHelper.create_table(db.connection, :test_table,
|
12
|
-
{'name' => 'character(128)', '
|
14
|
+
{'name' => 'character(128)', 'value' => 'integer'})
|
15
|
+
|
16
|
+
# id column created implicitly
|
17
|
+
PgSpecHelper.create_table(db.connection, :test_sub_table,
|
18
|
+
{'test_table_id' => 'integer', 'value' => 'character(128)', })
|
13
19
|
end
|
14
20
|
|
15
21
|
before(:each) do
|
16
22
|
PgSpecHelper.clear_table(db.connection, :test_table)
|
23
|
+
PgSpecHelper.clear_table(db.connection, :test_sub_table)
|
17
24
|
end
|
18
25
|
|
19
|
-
context
|
20
|
-
it
|
26
|
+
context 'A simple postgres database' do
|
27
|
+
it 'sets up a Sequel connection' do
|
21
28
|
expect(db.connection.class).to eq(Sequel::Postgres::Database)
|
22
29
|
end
|
23
30
|
|
24
|
-
context
|
31
|
+
context 'with 20 simple records' do
|
25
32
|
before(:each) do
|
26
33
|
(1..20).each do |i|
|
27
34
|
db.connection.run(
|
28
|
-
"insert into test_table (name,
|
35
|
+
"insert into test_table (name, value) values ('test', #{i})"
|
29
36
|
)
|
30
37
|
end
|
31
38
|
end
|
32
39
|
|
33
|
-
it
|
40
|
+
it 'can fetch records in batches' do
|
34
41
|
batch_sizes = []
|
35
42
|
db.records_in_batches(:test_table) do |batch|
|
36
43
|
batch_sizes << batch.size
|
@@ -38,29 +45,29 @@ describe PgShrink::Database::Postgres do
|
|
38
45
|
expect(batch_sizes).to eq([5, 5, 5, 5])
|
39
46
|
end
|
40
47
|
|
41
|
-
it
|
42
|
-
old_records = db.connection[
|
48
|
+
it 'throws an error if records change their primary keys during update' do
|
49
|
+
old_records = db.connection['select * from test_table where value <= 5'].
|
43
50
|
all
|
44
51
|
new_records = old_records.map {|r| r.merge(:id => r[:id] * 10)}
|
45
52
|
expect {db.update_records(:test_table, old_records, new_records)}.
|
46
53
|
to raise_error
|
47
54
|
end
|
48
55
|
|
49
|
-
it
|
50
|
-
db.delete_records(:test_table, {:
|
56
|
+
it 'can delete records based on a condition' do
|
57
|
+
db.delete_records(:test_table, {:value => 1..5})
|
51
58
|
|
52
|
-
results = db.connection[
|
59
|
+
results = db.connection['select * from test_table'].all
|
53
60
|
expect(results.size).to eq(15)
|
54
|
-
expect(results.map {|r| r[:
|
61
|
+
expect(results.map {|r| r[:value]}).to match_array((6..20).to_a)
|
55
62
|
end
|
56
63
|
|
57
|
-
it
|
58
|
-
old_records = db.connection[
|
64
|
+
it 'can update records' do
|
65
|
+
old_records = db.connection['select * from test_table where value <= 5'].
|
59
66
|
all
|
60
|
-
new_records = old_records.map {|r| r.merge(:
|
67
|
+
new_records = old_records.map {|r| r.merge(:value => r[:value] * 10)}
|
61
68
|
db.update_records(:test_table, old_records, new_records)
|
62
69
|
expect(
|
63
|
-
db.connection[
|
70
|
+
db.connection['select * from test_table where value <= 5'].all.size
|
64
71
|
).to eq(0)
|
65
72
|
updated_records = db.connection[:test_table].
|
66
73
|
where(:id => old_records.map {|r| r[:id]}).all
|
@@ -68,41 +75,121 @@ describe PgShrink::Database::Postgres do
|
|
68
75
|
expect(updated_records).to eq(new_records)
|
69
76
|
end
|
70
77
|
|
71
|
-
it
|
72
|
-
old_records = db.connection[
|
78
|
+
it 'throws an error if you try to delete records in update' do
|
79
|
+
old_records = db.connection['select * from test_table where value <= 5'].
|
73
80
|
all
|
74
81
|
new_records = old_records.first(2)
|
75
82
|
expect {db.update_records(:test_table, old_records, new_records)}.
|
76
83
|
to raise_error
|
77
84
|
end
|
78
85
|
|
79
|
-
it
|
86
|
+
it 'deletes the whole table' do
|
80
87
|
db.remove_table(:test_table)
|
81
88
|
db.filter!
|
82
|
-
expect(db.connection[
|
89
|
+
expect(db.connection['select * from test_table'].all.size).to eq(0)
|
83
90
|
end
|
84
|
-
|
91
|
+
|
92
|
+
|
93
|
+
describe 'on a table with no primary key' do
|
94
|
+
|
85
95
|
before(:all) do
|
86
96
|
PgSpecHelper.create_table(db.connection, :no_primary_key,
|
87
97
|
{'name' => 'character(128)',
|
88
98
|
'test' => 'integer'}, nil)
|
89
99
|
end
|
100
|
+
|
90
101
|
before(:each) do
|
102
|
+
PgSpecHelper.clear_table(db.connection, :no_primary_key)
|
91
103
|
(1..20).each do |i|
|
92
104
|
db.connection.run(
|
93
|
-
"insert into no_primary_key (name, test) values ('test', #{i})"
|
94
|
-
|
95
|
-
end
|
96
|
-
|
105
|
+
"insert into no_primary_key (name, test) values ('test', #{i})")
|
106
|
+
end
|
97
107
|
end
|
98
108
|
|
99
|
-
it
|
109
|
+
it 'can still remove the whole table' do
|
100
110
|
db.remove_table(:no_primary_key, :primary_key => false)
|
101
111
|
db.filter!
|
102
|
-
expect(db.connection[
|
112
|
+
expect(db.connection['select * from no_primary_key'].all.size).to eq(0)
|
103
113
|
|
104
|
-
|
114
|
+
end
|
115
|
+
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
context 'A table and subtable' do
|
122
|
+
|
123
|
+
before(:each) do
|
124
|
+
table_values = [
|
125
|
+
[1, 'john'],
|
126
|
+
[2, 'chris'],
|
127
|
+
[3, 'matt'],
|
128
|
+
]
|
129
|
+
|
130
|
+
sub_table_values = [
|
131
|
+
[1, 'john_value_1'],
|
132
|
+
[1, 'john_value_2'],
|
133
|
+
|
134
|
+
[2, 'chris_value_1'],
|
135
|
+
|
136
|
+
[3, 'matt_value_1'],
|
137
|
+
[3, 'matt_value_2'],
|
138
|
+
[3, 'matt_value_3'],
|
139
|
+
]
|
140
|
+
|
141
|
+
table_values.each do |row|
|
142
|
+
db.connection.run(
|
143
|
+
"insert into test_table (id, name, value) values (#{row[0]}, '#{row[1]}', #{row[0]})")
|
144
|
+
end
|
145
|
+
|
146
|
+
sub_table_values.each do |row|
|
147
|
+
db.connection.run(
|
148
|
+
"insert into test_sub_table (test_table_id, value) values (#{row[0]}, '#{row[1]}')")
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
context 'delete' do
|
153
|
+
|
154
|
+
before(:each) do
|
155
|
+
db.connection.run(
|
156
|
+
"delete from test_table where name = 'matt'"
|
157
|
+
)
|
158
|
+
end
|
159
|
+
|
160
|
+
it 'propagates to subtable' do
|
161
|
+
|
162
|
+
db.propagate_delete(
|
163
|
+
parent_table: 'test_table',
|
164
|
+
child_table: 'test_sub_table',
|
165
|
+
parent_key: 'id',
|
166
|
+
child_key: 'test_table_id')
|
167
|
+
|
168
|
+
expect(db.connection['select * from test_sub_table where test_table_id = 3'].all.size).to eq(0)
|
169
|
+
end
|
170
|
+
|
171
|
+
it 'propagates to subtable honoring where restriction' do
|
172
|
+
db.propagate_delete(
|
173
|
+
parent_table: 'test_table',
|
174
|
+
child_table: 'test_sub_table',
|
175
|
+
parent_key: 'id',
|
176
|
+
child_key: 'test_table_id',
|
177
|
+
where: "value = 'matt_value_2'")
|
178
|
+
|
179
|
+
expect(db.connection['select * from test_sub_table where test_table_id = 3'].all.size).to eq(2)
|
105
180
|
end
|
181
|
+
|
182
|
+
it 'propagates to subtable (nil where ignored)' do
|
183
|
+
db.propagate_delete(
|
184
|
+
parent_table: 'test_table',
|
185
|
+
child_table: 'test_sub_table',
|
186
|
+
parent_key: 'id',
|
187
|
+
child_key: 'test_table_id',
|
188
|
+
where: nil)
|
189
|
+
|
190
|
+
expect(db.connection['select * from test_sub_table where test_table_id = 3'].all.size).to eq(0)
|
191
|
+
end
|
192
|
+
|
106
193
|
end
|
107
194
|
end
|
108
195
|
end
|
@@ -45,14 +45,10 @@ describe PgShrink::Table do
|
|
45
45
|
end
|
46
46
|
end
|
47
47
|
|
48
|
-
context "when
|
48
|
+
context "when any subtable filter is specified" do
|
49
49
|
let(:database) {PgShrink::Database.new}
|
50
50
|
let(:table) { PgShrink::Table.new(database, :test_table, :primary_key => false) }
|
51
51
|
|
52
|
-
before(:each) do
|
53
|
-
table.filter_subtable(:subtable)
|
54
|
-
end
|
55
|
-
|
56
52
|
it "yields back a table so additional manipulations can be made" do
|
57
53
|
table.filter_subtable(:subtable) do |f|
|
58
54
|
expect(f.class).to eq(PgShrink::Table)
|
@@ -61,10 +57,12 @@ describe PgShrink::Table do
|
|
61
57
|
end
|
62
58
|
|
63
59
|
it "adds subtable_filter to subtable_filters array" do
|
60
|
+
table.filter_subtable(:subtable)
|
64
61
|
expect(table.subtable_filters.size).to eq(1)
|
65
62
|
end
|
66
63
|
end
|
67
64
|
|
65
|
+
|
68
66
|
context "when a remove is specified" do
|
69
67
|
let(:database) {PgShrink::Database.new}
|
70
68
|
let(:table) { PgShrink::Table.new(database, :test_table) }
|
data/spec/pg_shrink_spec.rb
CHANGED
@@ -14,13 +14,13 @@ describe PgShrink do
|
|
14
14
|
end
|
15
15
|
|
16
16
|
|
17
|
-
it
|
17
|
+
it 'should let you specify a different batch size' do
|
18
18
|
opts = PgSpecHelper.pg_config.merge(:batch_size => 20000)
|
19
19
|
db = PgShrink::Database::Postgres.new(opts)
|
20
20
|
expect(db.batch_size).to eq(20000)
|
21
21
|
end
|
22
22
|
|
23
|
-
describe
|
23
|
+
describe 'simple foreign_key setup' do
|
24
24
|
before(:all) do
|
25
25
|
# Rspec doesn't want you using 'let' defined things in before(:all)
|
26
26
|
connection = PgShrink::Database::Postgres.new(PgSpecHelper.pg_config).
|
@@ -35,9 +35,9 @@ describe PgShrink do
|
|
35
35
|
end
|
36
36
|
|
37
37
|
|
38
|
-
describe
|
38
|
+
describe 'simple two table filtering' do
|
39
39
|
|
40
|
-
describe
|
40
|
+
describe 'with 20 users and associated preferences' do
|
41
41
|
before(:each) do
|
42
42
|
PgSpecHelper.clear_table(database.connection, :users)
|
43
43
|
PgSpecHelper.clear_table(database.connection, :user_preferences)
|
@@ -53,24 +53,24 @@ describe PgShrink do
|
|
53
53
|
end
|
54
54
|
end
|
55
55
|
end
|
56
|
-
describe
|
56
|
+
describe 'with a conditions string' do
|
57
57
|
before(:each) do
|
58
58
|
database.filter_table(:users) do |f|
|
59
59
|
f.filter_by("name like '%test 1%'")
|
60
60
|
end
|
61
61
|
end
|
62
62
|
|
63
|
-
it
|
63
|
+
it 'Should not call records in batches' do
|
64
64
|
expect(database).not_to receive(:records_in_batches)
|
65
65
|
database.shrink!
|
66
66
|
end
|
67
67
|
|
68
|
-
it
|
68
|
+
it 'Should call delete_records once' do
|
69
69
|
expect(database).to receive(:delete_records).once
|
70
70
|
database.shrink!
|
71
71
|
end
|
72
72
|
|
73
|
-
it
|
73
|
+
it 'Should result in the appropriate records being deleted' do
|
74
74
|
database.shrink!
|
75
75
|
remaining_users = database.connection.from(:users).all
|
76
76
|
# 1 and 10-19
|
@@ -78,7 +78,7 @@ describe PgShrink do
|
|
78
78
|
end
|
79
79
|
end
|
80
80
|
|
81
|
-
describe
|
81
|
+
describe 'when filtering just with a conditions hash' do
|
82
82
|
before(:each) do
|
83
83
|
database.filter_table(:users) do |f|
|
84
84
|
f.filter_by({
|
@@ -88,30 +88,30 @@ describe PgShrink do
|
|
88
88
|
end
|
89
89
|
end
|
90
90
|
|
91
|
-
it
|
91
|
+
it 'Should not call records in batches' do
|
92
92
|
expect(database).not_to receive(:records_in_batches)
|
93
93
|
database.shrink!
|
94
94
|
end
|
95
95
|
|
96
|
-
it
|
96
|
+
it 'Should call delete_records once' do
|
97
97
|
expect(database).to receive(:delete_records).once
|
98
98
|
database.shrink!
|
99
99
|
end
|
100
100
|
|
101
|
-
it
|
101
|
+
it 'Should result in the appropriate records being deleted' do
|
102
102
|
database.shrink!
|
103
103
|
remaining_users = database.connection.from(:users).all
|
104
104
|
expect(remaining_users.size).to eq(10)
|
105
105
|
end
|
106
106
|
|
107
|
-
describe
|
107
|
+
describe 'with a subtable_filter' do
|
108
108
|
before(:each) do
|
109
109
|
database.filter_table(:users) do |f|
|
110
110
|
f.filter_subtable(:user_preferences, :foreign_key => :user_id)
|
111
111
|
end
|
112
112
|
end
|
113
113
|
|
114
|
-
it
|
114
|
+
it 'should remove the appropriate subtable records' do
|
115
115
|
database.shrink!
|
116
116
|
remaining_prefs = database.connection.from(:user_preferences).all
|
117
117
|
expect(remaining_prefs.size).to eq(30)
|
@@ -119,11 +119,11 @@ describe PgShrink do
|
|
119
119
|
end
|
120
120
|
end
|
121
121
|
|
122
|
-
describe
|
122
|
+
describe 'with a test shrinkfile' do
|
123
123
|
let(:shrinkfile) {"spec/Shrinkfile.basic"}
|
124
124
|
let(:url) {database.connection_string}
|
125
125
|
|
126
|
-
it
|
126
|
+
it 'should set up a postgres database' do
|
127
127
|
expect(PgShrink::Database::Postgres).to receive(:new) do |opts|
|
128
128
|
expect(opts[:postgres_url]).to eq(database.connection_string)
|
129
129
|
end.and_return(database)
|
@@ -132,7 +132,7 @@ describe PgShrink do
|
|
132
132
|
end
|
133
133
|
|
134
134
|
|
135
|
-
describe
|
135
|
+
describe 'a simple filter and subtable' do
|
136
136
|
before(:each) do
|
137
137
|
database.filter_table(:users) do |f|
|
138
138
|
f.filter_by "name = 'test 1'"
|
@@ -141,12 +141,12 @@ describe PgShrink do
|
|
141
141
|
database.filter!
|
142
142
|
end
|
143
143
|
|
144
|
-
it
|
144
|
+
it 'will filter users down to the one matching' do
|
145
145
|
remaining_users = database.connection.from(:users).all
|
146
146
|
expect(remaining_users.size).to eq(1)
|
147
147
|
end
|
148
148
|
|
149
|
-
it
|
149
|
+
it 'will filter preferences to only those associated with the user' do
|
150
150
|
remaining_user = database.connection.from(:users).first
|
151
151
|
remaining_preferences = database.connection.
|
152
152
|
from(:user_preferences).all
|
@@ -156,7 +156,7 @@ describe PgShrink do
|
|
156
156
|
end
|
157
157
|
end
|
158
158
|
|
159
|
-
describe
|
159
|
+
describe 'a simple filter and subtable with sanitization on each' do
|
160
160
|
|
161
161
|
before(:each) do
|
162
162
|
database.filter_table(:users) do |f|
|
@@ -179,14 +179,14 @@ describe PgShrink do
|
|
179
179
|
database.shrink!
|
180
180
|
end
|
181
181
|
|
182
|
-
it
|
182
|
+
it 'should result in 1 sanitized users' do
|
183
183
|
remaining_users = database.connection.from(:users).all
|
184
184
|
expect(remaining_users.size).to eq(1)
|
185
185
|
expect(remaining_users.first[:name]).to match(/sanitized/)
|
186
186
|
expect(remaining_users.first[:email]).to match(/blank_email/)
|
187
187
|
end
|
188
188
|
|
189
|
-
it
|
189
|
+
it 'should result in 3 sanitized preferences' do
|
190
190
|
remaining_user = database.connection.from(:users).first
|
191
191
|
remaining_preferences = database.connection.
|
192
192
|
from(:user_preferences).all
|
@@ -197,7 +197,7 @@ describe PgShrink do
|
|
197
197
|
end
|
198
198
|
end
|
199
199
|
end
|
200
|
-
describe
|
200
|
+
describe 'with users and preferences including email as value' do
|
201
201
|
before(:each) do
|
202
202
|
PgSpecHelper.clear_table(database.connection, :users)
|
203
203
|
PgSpecHelper.clear_table(database.connection, :user_preferences)
|
@@ -216,7 +216,7 @@ describe PgShrink do
|
|
216
216
|
end
|
217
217
|
end
|
218
218
|
|
219
|
-
describe
|
219
|
+
describe 'sanitizing subtable' do
|
220
220
|
before(:each) do
|
221
221
|
database.filter_table(:users) do |f|
|
222
222
|
f.sanitize do |u|
|
@@ -227,19 +227,18 @@ describe PgShrink do
|
|
227
227
|
:foreign_key => :user_id,
|
228
228
|
:local_field => :email,
|
229
229
|
:foreign_field => :value,
|
230
|
-
:
|
231
|
-
:type => 'email')
|
230
|
+
:where => { :name => 'email' })
|
232
231
|
end
|
233
232
|
database.shrink!
|
234
233
|
end
|
235
234
|
|
236
|
-
it
|
235
|
+
it 'should sanitize user preference emails' do
|
237
236
|
remaining_preferences = database.connection.
|
238
237
|
from(:user_preferences).where(:name => 'email').all
|
239
238
|
remaining_values = remaining_preferences.map {|p| p[:value]}
|
240
239
|
expect(remaining_values.grep(/blank_email/).size).to eq(20)
|
241
240
|
end
|
242
|
-
it
|
241
|
+
it 'should not sanitize preferences with a different type' do
|
243
242
|
remaining_preferences = database.connection.
|
244
243
|
from(:user_preferences).where(:name => 'name').all
|
245
244
|
remaining_values = remaining_preferences.map {|p| p[:value]}
|
@@ -250,7 +249,7 @@ describe PgShrink do
|
|
250
249
|
end
|
251
250
|
|
252
251
|
|
253
|
-
describe
|
252
|
+
describe 'three table filter chain' do
|
254
253
|
before(:all) do
|
255
254
|
# Rspec doesn't want you using 'let' defined things in before(:all)
|
256
255
|
connection = PgShrink::Database::Postgres.new(PgSpecHelper.pg_config).
|
@@ -260,7 +259,7 @@ describe PgShrink do
|
|
260
259
|
'character varying(256)'})
|
261
260
|
end
|
262
261
|
|
263
|
-
describe
|
262
|
+
describe 'with 20 users and associated preferences' do
|
264
263
|
before(:each) do
|
265
264
|
PgSpecHelper.clear_table(database.connection, :users)
|
266
265
|
PgSpecHelper.clear_table(database.connection, :user_preferences)
|
@@ -288,7 +287,7 @@ describe PgShrink do
|
|
288
287
|
end
|
289
288
|
end
|
290
289
|
|
291
|
-
describe
|
290
|
+
describe 'a simple filter and chained subtables' do
|
292
291
|
before(:each) do
|
293
292
|
database.filter_table(:users) do |f|
|
294
293
|
f.filter_by "name = 'test 1'"
|
@@ -301,11 +300,11 @@ describe PgShrink do
|
|
301
300
|
|
302
301
|
database.filter!
|
303
302
|
end
|
304
|
-
it
|
303
|
+
it 'filters users down to the one matching' do
|
305
304
|
remaining_users = database.connection.from(:users).all
|
306
305
|
expect(remaining_users.size).to eq(1)
|
307
306
|
end
|
308
|
-
it
|
307
|
+
it 'filters preferences to only those associated with the user' do
|
309
308
|
remaining_user = database.connection.from(:users).first
|
310
309
|
remaining_preferences = database.connection.
|
311
310
|
from(:user_preferences).all
|
@@ -313,8 +312,8 @@ describe PgShrink do
|
|
313
312
|
expect(remaining_preferences.map {|u| u[:user_id]}.uniq).
|
314
313
|
to eq([remaining_user[:id]])
|
315
314
|
end
|
316
|
-
it
|
317
|
-
|
315
|
+
it 'filters preference values to those associated with the ' +
|
316
|
+
'preferences remaining' do
|
318
317
|
remaining_user = database.connection.from(:users).first
|
319
318
|
remaining_preferences = database.connection.
|
320
319
|
from(:user_preferences).all
|
@@ -329,7 +328,7 @@ describe PgShrink do
|
|
329
328
|
end
|
330
329
|
end
|
331
330
|
end
|
332
|
-
describe
|
331
|
+
describe 'polymorphic foreign key subtables' do
|
333
332
|
before(:all) do
|
334
333
|
# Rspec doesn't want you using 'let' defined things in before(:all)
|
335
334
|
connection = PgShrink::Database::Postgres.new(PgSpecHelper.pg_config).
|
@@ -343,7 +342,7 @@ describe PgShrink do
|
|
343
342
|
'name' => 'character varying(256)',
|
344
343
|
'value' => 'character varying(256)'})
|
345
344
|
end
|
346
|
-
describe
|
345
|
+
describe 'with 20 users, associated prefs, and prefs for different type' do
|
347
346
|
before(:each) do
|
348
347
|
PgSpecHelper.clear_table(database.connection, :users)
|
349
348
|
PgSpecHelper.clear_table(database.connection, :preferences)
|
@@ -363,17 +362,17 @@ describe PgShrink do
|
|
363
362
|
end
|
364
363
|
end
|
365
364
|
|
366
|
-
describe
|
367
|
-
describe
|
365
|
+
describe 'with condition filters' do
|
366
|
+
describe 'simple cascade' do
|
368
367
|
before(:each) do
|
369
368
|
database.filter_table(:users) do |f|
|
370
|
-
f.filter_by(:
|
371
|
-
f.filter_subtable(:preferences, :
|
372
|
-
|
369
|
+
f.filter_by(name: 'test 1')
|
370
|
+
f.filter_subtable(:preferences, foreign_key: :context_id,
|
371
|
+
where: { context_type: 'User' } )
|
373
372
|
end
|
374
373
|
database.filter!
|
375
374
|
end
|
376
|
-
it
|
375
|
+
it 'will filter prefs with context_type User' do
|
377
376
|
#
|
378
377
|
remaining_user = database.connection.from(:users).first
|
379
378
|
remaining_preferences = database.connection.from(:preferences).
|
@@ -383,13 +382,13 @@ describe PgShrink do
|
|
383
382
|
to eq([remaining_user[:id]])
|
384
383
|
end
|
385
384
|
|
386
|
-
it
|
385
|
+
it 'will not filter preferences without context_type user' do
|
387
386
|
remaining_preferences = database.connection.from(:preferences).
|
388
387
|
where(:context_type => 'OtherClass').all
|
389
388
|
expect(remaining_preferences.size).to eq(20)
|
390
389
|
end
|
391
390
|
end
|
392
|
-
describe
|
391
|
+
describe 'an extra layer of polymorphic subtables' do
|
393
392
|
before(:all) do
|
394
393
|
connection = PgShrink::Database::Postgres.new(PgSpecHelper.pg_config).
|
395
394
|
connection
|
@@ -416,20 +415,19 @@ describe PgShrink do
|
|
416
415
|
end
|
417
416
|
|
418
417
|
database.filter_table(:users) do |f|
|
419
|
-
f.filter_by(:
|
420
|
-
f.filter_subtable(:preferences, :
|
421
|
-
|
418
|
+
f.filter_by(name: 'test 1')
|
419
|
+
f.filter_subtable(:preferences, foreign_key: :context_id,
|
420
|
+
where: { context_type: 'User' } )
|
422
421
|
end
|
423
422
|
|
424
423
|
database.filter_table(:preferences) do |f|
|
425
424
|
f.filter_subtable(:preference_dependents,
|
426
|
-
:
|
427
|
-
|
428
|
-
:type => 'Preference')
|
425
|
+
foreign_key: :context_id,
|
426
|
+
where: { context_type: 'Preference' })
|
429
427
|
end
|
430
428
|
database.filter!
|
431
429
|
end
|
432
|
-
it
|
430
|
+
it 'will filter preference dependents associated with preferences' do
|
433
431
|
remaining_preferences = database.connection.from(:preferences).all
|
434
432
|
remaining_dependents = database.connection.
|
435
433
|
from(:preference_dependents).
|
@@ -438,7 +436,7 @@ describe PgShrink do
|
|
438
436
|
expect(remaining_dependents.size).to eq(remaining_preferences.size)
|
439
437
|
end
|
440
438
|
|
441
|
-
it
|
439
|
+
it 'will not filter preference dependents with different type' do
|
442
440
|
other_dependents = database.connection.
|
443
441
|
from(:preference_dependents).
|
444
442
|
where(:context_type => 'SomeOtherClass').all
|
@@ -447,17 +445,17 @@ describe PgShrink do
|
|
447
445
|
end
|
448
446
|
end
|
449
447
|
|
450
|
-
describe
|
448
|
+
describe 'simple two table filtering' do
|
451
449
|
before(:each) do
|
452
450
|
database.filter_table(:users) do |f|
|
453
451
|
f.filter_by "name = 'test 1'"
|
454
|
-
f.filter_subtable(:preferences, :
|
455
|
-
|
452
|
+
f.filter_subtable(:preferences, foreign_key: :context_id,
|
453
|
+
where: {context_type: 'User'})
|
456
454
|
end
|
457
455
|
database.filter!
|
458
456
|
end
|
459
457
|
|
460
|
-
it
|
458
|
+
it 'will filter prefs with context_type User' do
|
461
459
|
remaining_user = database.connection.from(:users).first
|
462
460
|
remaining_preferences = database.connection.from(:preferences).
|
463
461
|
where(:context_type => 'User').all
|
@@ -466,14 +464,14 @@ describe PgShrink do
|
|
466
464
|
to eq([remaining_user[:id]])
|
467
465
|
end
|
468
466
|
|
469
|
-
it
|
467
|
+
it 'will not filter preferences without context_type user' do
|
470
468
|
remaining_preferences = database.connection.from(:preferences).
|
471
469
|
where(:context_type => 'OtherClass').all
|
472
470
|
expect(remaining_preferences.size).to eq(20)
|
473
471
|
end
|
474
472
|
end
|
475
473
|
|
476
|
-
describe
|
474
|
+
describe 'an extra layer of polymorphic subtables' do
|
477
475
|
before(:all) do
|
478
476
|
connection = PgShrink::Database::Postgres.new(PgSpecHelper.pg_config).
|
479
477
|
connection
|
@@ -501,20 +499,19 @@ describe PgShrink do
|
|
501
499
|
|
502
500
|
database.filter_table(:users) do |f|
|
503
501
|
f.filter_by "name = 'test 1'"
|
504
|
-
f.filter_subtable(:preferences, :
|
505
|
-
|
502
|
+
f.filter_subtable(:preferences, foreign_key: :context_id,
|
503
|
+
where: {context_type: 'User'} )
|
506
504
|
end
|
507
505
|
|
508
506
|
database.filter_table(:preferences) do |f|
|
509
507
|
f.filter_subtable(:preference_dependents,
|
510
|
-
:
|
511
|
-
|
512
|
-
:type => 'Preference')
|
508
|
+
foreign_key: :context_id,
|
509
|
+
where: {context_type: 'Preference'} )
|
513
510
|
end
|
514
511
|
database.filter!
|
515
512
|
end
|
516
513
|
|
517
|
-
it
|
514
|
+
it 'will filter preference dependents associated with preferences' do
|
518
515
|
remaining_preferences = database.connection.from(:preferences).all
|
519
516
|
remaining_dependents = database.connection.
|
520
517
|
from(:preference_dependents).
|
@@ -523,7 +520,7 @@ describe PgShrink do
|
|
523
520
|
expect(remaining_dependents.size).to eq(remaining_preferences.size)
|
524
521
|
end
|
525
522
|
|
526
|
-
it
|
523
|
+
it 'will not filter preference dependents with different type' do
|
527
524
|
other_dependents = database.connection.
|
528
525
|
from(:preference_dependents).
|
529
526
|
where(:context_type => 'SomeOtherClass').all
|
@@ -532,7 +529,7 @@ describe PgShrink do
|
|
532
529
|
end
|
533
530
|
end
|
534
531
|
end
|
535
|
-
describe
|
532
|
+
describe 'has_and_belongs_to_many join tables' do
|
536
533
|
before(:all) do
|
537
534
|
# Rspec doesn't want you using 'let' defined things in before(;all)
|
538
535
|
connection = PgShrink::Database::Postgres.new(PgSpecHelper.pg_config).
|
@@ -547,7 +544,7 @@ describe PgShrink do
|
|
547
544
|
{'name' => 'character varying(256)'})
|
548
545
|
end
|
549
546
|
|
550
|
-
describe
|
547
|
+
describe 'with 5 users, each with 2 apartments, and 1 apartment shared by all 5' do
|
551
548
|
before(:each) do
|
552
549
|
PgSpecHelper.clear_table(database.connection, :users)
|
553
550
|
PgSpecHelper.clear_table(database.connection, :apartments_users)
|
@@ -572,7 +569,7 @@ describe PgShrink do
|
|
572
569
|
end
|
573
570
|
end
|
574
571
|
end
|
575
|
-
describe
|
572
|
+
describe 'With a simple cascading filter' do
|
576
573
|
before(:each) do
|
577
574
|
database.filter_table(:users) do |f|
|
578
575
|
f.filter_by "name = 'test 1'"
|
@@ -586,13 +583,13 @@ describe PgShrink do
|
|
586
583
|
database.shrink!
|
587
584
|
end
|
588
585
|
|
589
|
-
it
|
586
|
+
it 'Should filter down apartments_users' do
|
590
587
|
u = database.connection.from(:users).where(:name => "test 1").first
|
591
588
|
remaining_join_table = database.connection.from(:apartments_users).all
|
592
589
|
expect(remaining_join_table.size).to eq(3)
|
593
590
|
end
|
594
591
|
|
595
|
-
it
|
592
|
+
it 'Should filter apartments as well' do
|
596
593
|
remaining_join_table = database.connection.from(:apartments_users).all
|
597
594
|
remaining_apartments = database.connection.from(:apartments).all
|
598
595
|
expect(remaining_apartments.size).to eq(3)
|
metadata
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_shrink
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kevin Ball
|
8
|
+
- Matt Nemenman
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
date: 2014-
|
12
|
+
date: 2014-07-10 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
13
14
|
- !ruby/object:Gem::Dependency
|
14
15
|
name: pg
|
@@ -195,6 +196,7 @@ dependencies:
|
|
195
196
|
description: pg_shrink makes it simple to shrink and sanitize a PosrgreSQL database
|
196
197
|
email:
|
197
198
|
- kmball11@gmail.com
|
199
|
+
- matt@apartmentlist.com
|
198
200
|
executables:
|
199
201
|
- pg_shrink
|
200
202
|
extensions: []
|
@@ -229,7 +231,7 @@ files:
|
|
229
231
|
- spec/pg_shrink_spec.rb
|
230
232
|
- spec/pg_spec_helper.rb
|
231
233
|
- spec/spec_helper.rb
|
232
|
-
homepage: https://github.com/apartmentlist/
|
234
|
+
homepage: https://github.com/apartmentlist/pg_shrink
|
233
235
|
licenses:
|
234
236
|
- MIT
|
235
237
|
metadata: {}
|