pg_shrink 0.0.7 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -2
- data/Shrinkfile.example +10 -5
- data/lib/pg_shrink/database/postgres.rb +15 -13
- data/lib/pg_shrink/sub_table_filter.rb +11 -28
- data/lib/pg_shrink/sub_table_operator.rb +5 -11
- data/lib/pg_shrink/sub_table_sanitizer.rb +4 -3
- data/lib/pg_shrink/version.rb +1 -1
- data/pg_shrink.gemspec +3 -3
- data/spec/pg_shrink/database/postgres_spec.rb +115 -28
- data/spec/pg_shrink/table_spec.rb +3 -5
- data/spec/pg_shrink_spec.rb +68 -71
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 79d28599347bf3d3b21a5ff1066eb005d93de451
|
4
|
+
data.tar.gz: 52afef4d72dfe607d9a6b9fea197789492a27153
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b34ea492d943779dafde384932065fdaae0d5244ff09b115c5436a319c27bdccae5ac01c02dcfda578a54a3383b725f561c5333fc199efd8e91dae81ad542303
|
7
|
+
data.tar.gz: b3c7736462f971e4eba50960a48dea9260c891d1b0152c7fe675669213ac241e459a9dce38f6a32cdfb8e5765d17a898e78ed756aa95abe4476cb233f8aad103
|
data/README.md
CHANGED
@@ -1,5 +1,4 @@
|
|
1
|
-
[](https://travis-ci.org/apartmentlist/pg_shrink)
|
3
2
|
# PgShrink
|
4
3
|
|
5
4
|
The pg_shrink tool makes it easy to shrink and sanitize a postgres database,
|
data/Shrinkfile.example
CHANGED
@@ -23,11 +23,16 @@ filter_table :users do |f|
|
|
23
23
|
f.filter_subtable(:email_preferences, :foreign_key => :user_email,
|
24
24
|
:primary_key => :email)
|
25
25
|
|
26
|
-
# You can also
|
27
|
-
#
|
28
|
-
|
29
|
-
|
30
|
-
|
26
|
+
# You can also pass an additional where clause expression that restricts
|
27
|
+
# filtering propagation to certain subset of rows. This allows to support
|
28
|
+
# polymorphic relationships, or STI.
|
29
|
+
f.filter_subtable(:email_preferences, foreign_key: user_id,
|
30
|
+
where: "type = 'User::Registered'")
|
31
|
+
|
32
|
+
# 'where' clause can be anything that sequel supports
|
33
|
+
f.filter_subtable(:email_preferences, foreign_key: user_id,
|
34
|
+
where: { type: 'User::Registered',
|
35
|
+
color: 'Red'})
|
31
36
|
|
32
37
|
# If it feels more natural, you can define additional filters
|
33
38
|
# within a filter_subtable definitition
|
@@ -113,31 +113,33 @@ module PgShrink
|
|
113
113
|
end
|
114
114
|
|
115
115
|
def propagate_delete(opts)
|
116
|
+
|
117
|
+
child_table = opts[:child_table]
|
118
|
+
parent_table = opts[:parent_table]
|
119
|
+
child_key = opts[:child_key]
|
120
|
+
parent_key = opts[:parent_key]
|
121
|
+
where_clause = opts[:where]
|
122
|
+
|
116
123
|
# what we conceptually want to do is delete the left outer join where id is null.
|
117
124
|
# That's not working in postgres, so we instead use where not exists. Docs
|
118
125
|
# indicate using where not exists and select 1 in this case.
|
119
126
|
# See:
|
120
127
|
# http://www.postgresql.org/docs/current/interactive/functions-subquery.html#FUNCTIONS-SUBQUERY-EXISTS
|
121
|
-
query = "DELETE FROM #{
|
122
|
-
"SELECT 1 from #{
|
123
|
-
"#{
|
124
|
-
"#{
|
128
|
+
query = "DELETE FROM #{child_table} WHERE NOT EXISTS (" +
|
129
|
+
"SELECT 1 from #{parent_table} where " +
|
130
|
+
"#{child_table}.#{child_key} = " +
|
131
|
+
"#{parent_table}.#{parent_key}" +
|
125
132
|
")"
|
126
133
|
|
134
|
+
query_builder = connection.from(child_table)
|
135
|
+
query_builder = query_builder.where(where_clause) if where_clause
|
127
136
|
|
128
137
|
# Outside of the join statements, we want to maintain the ease of hash-based
|
129
138
|
# conditions. Do this by using a query builder but then swapping in delete SQL
|
130
139
|
# in the end.
|
131
|
-
|
132
|
-
Array.wrap(opts[:conditions]).compact.each do |cond|
|
133
|
-
query_builder = query_builder.where(cond)
|
134
|
-
end
|
135
|
-
Array.wrap(opts[:exclude]).compact.each do |exclude_cond|
|
136
|
-
query_builder = query_builder.exclude(exclude_cond)
|
137
|
-
end
|
140
|
+
|
138
141
|
sql = query_builder.sql.gsub("WHERE", "AND").
|
139
|
-
gsub("SELECT * FROM \"#{
|
140
|
-
query)
|
142
|
+
gsub("SELECT * FROM \"#{child_table}\"", query)
|
141
143
|
|
142
144
|
connection[sql].delete
|
143
145
|
end
|
@@ -4,42 +4,25 @@ module PgShrink
|
|
4
4
|
def propagate_table!
|
5
5
|
primary_key = @opts[:primary_key]
|
6
6
|
foreign_key = @opts[:foreign_key]
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
end
|
11
|
-
self.database.log("Beginning subtable propagation from " +
|
7
|
+
where_clause = @opts[:where]
|
8
|
+
|
9
|
+
self.database.log('Beginning subtable propagation from ' +
|
12
10
|
"#{self.parent.table_name} to #{self.table.table_name}")
|
13
|
-
self.database.propagate_delete(:parent_table => self.parent.table_name,
|
14
|
-
:child_table => self.table.table_name,
|
15
|
-
:parent_key => primary_key,
|
16
|
-
:child_key => foreign_key,
|
17
|
-
:conditions => additional_conditions)
|
18
11
|
|
19
|
-
self.database.
|
12
|
+
self.database.propagate_delete(parent_table: self.parent.table_name,
|
13
|
+
child_table: self.table.table_name,
|
14
|
+
parent_key: primary_key,
|
15
|
+
child_key: foreign_key,
|
16
|
+
where: where_clause)
|
17
|
+
|
18
|
+
self.database.log('Done with subtable propagation from ' +
|
20
19
|
"#{self.parent.table_name} to #{self.table.table_name}")
|
20
|
+
|
21
21
|
if self.table.subtable_filters.any?
|
22
22
|
self.database.vacuum_and_reindex!(self.table.table_name)
|
23
23
|
self.table.subtable_filters.each(&:propagate_table!)
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
27
|
-
def propagate!(old_parent_data, new_parent_data)
|
28
|
-
return if (old_parent_data.empty? && new_parent_data.empty?)
|
29
|
-
old_batch_keys = old_parent_data.map {|record| record[@opts[:primary_key]]}
|
30
|
-
new_batch_keys = new_parent_data.map {|record| record[@opts[:primary_key]]}
|
31
|
-
|
32
|
-
foreign_key = @opts[:foreign_key]
|
33
|
-
finder_options = {foreign_key => old_batch_keys}
|
34
|
-
if @opts[:type_key] && @opts[:type]
|
35
|
-
finder_options[@opts[:type_key]] = @opts[:type]
|
36
|
-
end
|
37
|
-
|
38
|
-
old_records = table.get_records(finder_options)
|
39
|
-
table.filter_batch(old_records) do |record|
|
40
|
-
new_batch_keys.include?(record[foreign_key])
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
27
|
end
|
45
28
|
end
|
@@ -2,9 +2,10 @@ module PgShrink
|
|
2
2
|
class SubTableOperator
|
3
3
|
attr_accessor :parent, :table_name, :database
|
4
4
|
def default_opts
|
5
|
-
{
|
5
|
+
{
|
6
|
+
:foreign_key =>
|
6
7
|
"#{ActiveSupport::Inflector.singularize(parent.table_name.to_s)}_id",
|
7
|
-
|
8
|
+
:primary_key => :id
|
8
9
|
}
|
9
10
|
end
|
10
11
|
|
@@ -16,15 +17,6 @@ module PgShrink
|
|
16
17
|
database.table(table_name)
|
17
18
|
end
|
18
19
|
|
19
|
-
def validate_opts!(opts)
|
20
|
-
if opts[:type_key] && !opts[:type]
|
21
|
-
raise "Error: #{name} has type_key set but no type"
|
22
|
-
end
|
23
|
-
if opts[:type] && !opts[:type_key]
|
24
|
-
raise "Error: #{name} has type set but no type_key"
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
20
|
def initialize(parent, table_name, opts = {})
|
29
21
|
self.parent = parent
|
30
22
|
self.table_name = table_name
|
@@ -34,6 +26,8 @@ module PgShrink
|
|
34
26
|
validate_opts!(@opts)
|
35
27
|
end
|
36
28
|
|
29
|
+
def validate_opts!(opts); end
|
30
|
+
|
37
31
|
def propagate!(old_parent_data, new_parent_data)
|
38
32
|
raise "Implement in subclass"
|
39
33
|
end
|
@@ -15,9 +15,10 @@ module PgShrink
|
|
15
15
|
|
16
16
|
foreign_key = @opts[:foreign_key]
|
17
17
|
finder_options = {foreign_key => old_batch.keys}
|
18
|
-
|
19
|
-
|
20
|
-
|
18
|
+
|
19
|
+
# adding additional filters from where clause.
|
20
|
+
# TODO: support where clauses using non hash syntax (string, symbol)
|
21
|
+
finder_options.merge!(@opts[:where])
|
21
22
|
|
22
23
|
parent_field = @opts[:local_field].to_sym
|
23
24
|
child_field = @opts[:foreign_field].to_sym
|
data/lib/pg_shrink/version.rb
CHANGED
data/pg_shrink.gemspec
CHANGED
@@ -6,11 +6,11 @@ require 'pg_shrink/version'
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = 'pg_shrink'
|
8
8
|
spec.version = PgShrink::VERSION
|
9
|
-
spec.authors = ['Kevin Ball']
|
10
|
-
spec.email = ['kmball11@gmail.com']
|
9
|
+
spec.authors = ['Kevin Ball', 'Matt Nemenman']
|
10
|
+
spec.email = ['kmball11@gmail.com', 'matt@apartmentlist.com']
|
11
11
|
spec.description = 'pg_shrink makes it simple to shrink and sanitize a PosrgreSQL database'
|
12
12
|
spec.summary = 'pg_shrink'
|
13
|
-
spec.homepage = 'https://github.com/apartmentlist/
|
13
|
+
spec.homepage = 'https://github.com/apartmentlist/pg_shrink'
|
14
14
|
spec.license = 'MIT'
|
15
15
|
|
16
16
|
spec.files = `git ls-files`.split($/)
|
@@ -8,29 +8,36 @@ describe PgShrink::Database::Postgres do
|
|
8
8
|
|
9
9
|
before(:all) do
|
10
10
|
PgSpecHelper.reset_database
|
11
|
+
|
12
|
+
# id column created implicitly
|
11
13
|
PgSpecHelper.create_table(db.connection, :test_table,
|
12
|
-
{'name' => 'character(128)', '
|
14
|
+
{'name' => 'character(128)', 'value' => 'integer'})
|
15
|
+
|
16
|
+
# id column created implicitly
|
17
|
+
PgSpecHelper.create_table(db.connection, :test_sub_table,
|
18
|
+
{'test_table_id' => 'integer', 'value' => 'character(128)', })
|
13
19
|
end
|
14
20
|
|
15
21
|
before(:each) do
|
16
22
|
PgSpecHelper.clear_table(db.connection, :test_table)
|
23
|
+
PgSpecHelper.clear_table(db.connection, :test_sub_table)
|
17
24
|
end
|
18
25
|
|
19
|
-
context
|
20
|
-
it
|
26
|
+
context 'A simple postgres database' do
|
27
|
+
it 'sets up a Sequel connection' do
|
21
28
|
expect(db.connection.class).to eq(Sequel::Postgres::Database)
|
22
29
|
end
|
23
30
|
|
24
|
-
context
|
31
|
+
context 'with 20 simple records' do
|
25
32
|
before(:each) do
|
26
33
|
(1..20).each do |i|
|
27
34
|
db.connection.run(
|
28
|
-
"insert into test_table (name,
|
35
|
+
"insert into test_table (name, value) values ('test', #{i})"
|
29
36
|
)
|
30
37
|
end
|
31
38
|
end
|
32
39
|
|
33
|
-
it
|
40
|
+
it 'can fetch records in batches' do
|
34
41
|
batch_sizes = []
|
35
42
|
db.records_in_batches(:test_table) do |batch|
|
36
43
|
batch_sizes << batch.size
|
@@ -38,29 +45,29 @@ describe PgShrink::Database::Postgres do
|
|
38
45
|
expect(batch_sizes).to eq([5, 5, 5, 5])
|
39
46
|
end
|
40
47
|
|
41
|
-
it
|
42
|
-
old_records = db.connection[
|
48
|
+
it 'throws an error if records change their primary keys during update' do
|
49
|
+
old_records = db.connection['select * from test_table where value <= 5'].
|
43
50
|
all
|
44
51
|
new_records = old_records.map {|r| r.merge(:id => r[:id] * 10)}
|
45
52
|
expect {db.update_records(:test_table, old_records, new_records)}.
|
46
53
|
to raise_error
|
47
54
|
end
|
48
55
|
|
49
|
-
it
|
50
|
-
db.delete_records(:test_table, {:
|
56
|
+
it 'can delete records based on a condition' do
|
57
|
+
db.delete_records(:test_table, {:value => 1..5})
|
51
58
|
|
52
|
-
results = db.connection[
|
59
|
+
results = db.connection['select * from test_table'].all
|
53
60
|
expect(results.size).to eq(15)
|
54
|
-
expect(results.map {|r| r[:
|
61
|
+
expect(results.map {|r| r[:value]}).to match_array((6..20).to_a)
|
55
62
|
end
|
56
63
|
|
57
|
-
it
|
58
|
-
old_records = db.connection[
|
64
|
+
it 'can update records' do
|
65
|
+
old_records = db.connection['select * from test_table where value <= 5'].
|
59
66
|
all
|
60
|
-
new_records = old_records.map {|r| r.merge(:
|
67
|
+
new_records = old_records.map {|r| r.merge(:value => r[:value] * 10)}
|
61
68
|
db.update_records(:test_table, old_records, new_records)
|
62
69
|
expect(
|
63
|
-
db.connection[
|
70
|
+
db.connection['select * from test_table where value <= 5'].all.size
|
64
71
|
).to eq(0)
|
65
72
|
updated_records = db.connection[:test_table].
|
66
73
|
where(:id => old_records.map {|r| r[:id]}).all
|
@@ -68,41 +75,121 @@ describe PgShrink::Database::Postgres do
|
|
68
75
|
expect(updated_records).to eq(new_records)
|
69
76
|
end
|
70
77
|
|
71
|
-
it
|
72
|
-
old_records = db.connection[
|
78
|
+
it 'throws an error if you try to delete records in update' do
|
79
|
+
old_records = db.connection['select * from test_table where value <= 5'].
|
73
80
|
all
|
74
81
|
new_records = old_records.first(2)
|
75
82
|
expect {db.update_records(:test_table, old_records, new_records)}.
|
76
83
|
to raise_error
|
77
84
|
end
|
78
85
|
|
79
|
-
it
|
86
|
+
it 'deletes the whole table' do
|
80
87
|
db.remove_table(:test_table)
|
81
88
|
db.filter!
|
82
|
-
expect(db.connection[
|
89
|
+
expect(db.connection['select * from test_table'].all.size).to eq(0)
|
83
90
|
end
|
84
|
-
|
91
|
+
|
92
|
+
|
93
|
+
describe 'on a table with no primary key' do
|
94
|
+
|
85
95
|
before(:all) do
|
86
96
|
PgSpecHelper.create_table(db.connection, :no_primary_key,
|
87
97
|
{'name' => 'character(128)',
|
88
98
|
'test' => 'integer'}, nil)
|
89
99
|
end
|
100
|
+
|
90
101
|
before(:each) do
|
102
|
+
PgSpecHelper.clear_table(db.connection, :no_primary_key)
|
91
103
|
(1..20).each do |i|
|
92
104
|
db.connection.run(
|
93
|
-
"insert into no_primary_key (name, test) values ('test', #{i})"
|
94
|
-
|
95
|
-
end
|
96
|
-
|
105
|
+
"insert into no_primary_key (name, test) values ('test', #{i})")
|
106
|
+
end
|
97
107
|
end
|
98
108
|
|
99
|
-
it
|
109
|
+
it 'can still remove the whole table' do
|
100
110
|
db.remove_table(:no_primary_key, :primary_key => false)
|
101
111
|
db.filter!
|
102
|
-
expect(db.connection[
|
112
|
+
expect(db.connection['select * from no_primary_key'].all.size).to eq(0)
|
103
113
|
|
104
|
-
|
114
|
+
end
|
115
|
+
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
context 'A table and subtable' do
|
122
|
+
|
123
|
+
before(:each) do
|
124
|
+
table_values = [
|
125
|
+
[1, 'john'],
|
126
|
+
[2, 'chris'],
|
127
|
+
[3, 'matt'],
|
128
|
+
]
|
129
|
+
|
130
|
+
sub_table_values = [
|
131
|
+
[1, 'john_value_1'],
|
132
|
+
[1, 'john_value_2'],
|
133
|
+
|
134
|
+
[2, 'chris_value_1'],
|
135
|
+
|
136
|
+
[3, 'matt_value_1'],
|
137
|
+
[3, 'matt_value_2'],
|
138
|
+
[3, 'matt_value_3'],
|
139
|
+
]
|
140
|
+
|
141
|
+
table_values.each do |row|
|
142
|
+
db.connection.run(
|
143
|
+
"insert into test_table (id, name, value) values (#{row[0]}, '#{row[1]}', #{row[0]})")
|
144
|
+
end
|
145
|
+
|
146
|
+
sub_table_values.each do |row|
|
147
|
+
db.connection.run(
|
148
|
+
"insert into test_sub_table (test_table_id, value) values (#{row[0]}, '#{row[1]}')")
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
context 'delete' do
|
153
|
+
|
154
|
+
before(:each) do
|
155
|
+
db.connection.run(
|
156
|
+
"delete from test_table where name = 'matt'"
|
157
|
+
)
|
158
|
+
end
|
159
|
+
|
160
|
+
it 'propagates to subtable' do
|
161
|
+
|
162
|
+
db.propagate_delete(
|
163
|
+
parent_table: 'test_table',
|
164
|
+
child_table: 'test_sub_table',
|
165
|
+
parent_key: 'id',
|
166
|
+
child_key: 'test_table_id')
|
167
|
+
|
168
|
+
expect(db.connection['select * from test_sub_table where test_table_id = 3'].all.size).to eq(0)
|
169
|
+
end
|
170
|
+
|
171
|
+
it 'propagates to subtable honoring where restriction' do
|
172
|
+
db.propagate_delete(
|
173
|
+
parent_table: 'test_table',
|
174
|
+
child_table: 'test_sub_table',
|
175
|
+
parent_key: 'id',
|
176
|
+
child_key: 'test_table_id',
|
177
|
+
where: "value = 'matt_value_2'")
|
178
|
+
|
179
|
+
expect(db.connection['select * from test_sub_table where test_table_id = 3'].all.size).to eq(2)
|
105
180
|
end
|
181
|
+
|
182
|
+
it 'propagates to subtable (nil where ignored)' do
|
183
|
+
db.propagate_delete(
|
184
|
+
parent_table: 'test_table',
|
185
|
+
child_table: 'test_sub_table',
|
186
|
+
parent_key: 'id',
|
187
|
+
child_key: 'test_table_id',
|
188
|
+
where: nil)
|
189
|
+
|
190
|
+
expect(db.connection['select * from test_sub_table where test_table_id = 3'].all.size).to eq(0)
|
191
|
+
end
|
192
|
+
|
106
193
|
end
|
107
194
|
end
|
108
195
|
end
|
@@ -45,14 +45,10 @@ describe PgShrink::Table do
|
|
45
45
|
end
|
46
46
|
end
|
47
47
|
|
48
|
-
context "when
|
48
|
+
context "when any subtable filter is specified" do
|
49
49
|
let(:database) {PgShrink::Database.new}
|
50
50
|
let(:table) { PgShrink::Table.new(database, :test_table, :primary_key => false) }
|
51
51
|
|
52
|
-
before(:each) do
|
53
|
-
table.filter_subtable(:subtable)
|
54
|
-
end
|
55
|
-
|
56
52
|
it "yields back a table so additional manipulations can be made" do
|
57
53
|
table.filter_subtable(:subtable) do |f|
|
58
54
|
expect(f.class).to eq(PgShrink::Table)
|
@@ -61,10 +57,12 @@ describe PgShrink::Table do
|
|
61
57
|
end
|
62
58
|
|
63
59
|
it "adds subtable_filter to subtable_filters array" do
|
60
|
+
table.filter_subtable(:subtable)
|
64
61
|
expect(table.subtable_filters.size).to eq(1)
|
65
62
|
end
|
66
63
|
end
|
67
64
|
|
65
|
+
|
68
66
|
context "when a remove is specified" do
|
69
67
|
let(:database) {PgShrink::Database.new}
|
70
68
|
let(:table) { PgShrink::Table.new(database, :test_table) }
|
data/spec/pg_shrink_spec.rb
CHANGED
@@ -14,13 +14,13 @@ describe PgShrink do
|
|
14
14
|
end
|
15
15
|
|
16
16
|
|
17
|
-
it
|
17
|
+
it 'should let you specify a different batch size' do
|
18
18
|
opts = PgSpecHelper.pg_config.merge(:batch_size => 20000)
|
19
19
|
db = PgShrink::Database::Postgres.new(opts)
|
20
20
|
expect(db.batch_size).to eq(20000)
|
21
21
|
end
|
22
22
|
|
23
|
-
describe
|
23
|
+
describe 'simple foreign_key setup' do
|
24
24
|
before(:all) do
|
25
25
|
# Rspec doesn't want you using 'let' defined things in before(:all)
|
26
26
|
connection = PgShrink::Database::Postgres.new(PgSpecHelper.pg_config).
|
@@ -35,9 +35,9 @@ describe PgShrink do
|
|
35
35
|
end
|
36
36
|
|
37
37
|
|
38
|
-
describe
|
38
|
+
describe 'simple two table filtering' do
|
39
39
|
|
40
|
-
describe
|
40
|
+
describe 'with 20 users and associated preferences' do
|
41
41
|
before(:each) do
|
42
42
|
PgSpecHelper.clear_table(database.connection, :users)
|
43
43
|
PgSpecHelper.clear_table(database.connection, :user_preferences)
|
@@ -53,24 +53,24 @@ describe PgShrink do
|
|
53
53
|
end
|
54
54
|
end
|
55
55
|
end
|
56
|
-
describe
|
56
|
+
describe 'with a conditions string' do
|
57
57
|
before(:each) do
|
58
58
|
database.filter_table(:users) do |f|
|
59
59
|
f.filter_by("name like '%test 1%'")
|
60
60
|
end
|
61
61
|
end
|
62
62
|
|
63
|
-
it
|
63
|
+
it 'Should not call records in batches' do
|
64
64
|
expect(database).not_to receive(:records_in_batches)
|
65
65
|
database.shrink!
|
66
66
|
end
|
67
67
|
|
68
|
-
it
|
68
|
+
it 'Should call delete_records once' do
|
69
69
|
expect(database).to receive(:delete_records).once
|
70
70
|
database.shrink!
|
71
71
|
end
|
72
72
|
|
73
|
-
it
|
73
|
+
it 'Should result in the appropriate records being deleted' do
|
74
74
|
database.shrink!
|
75
75
|
remaining_users = database.connection.from(:users).all
|
76
76
|
# 1 and 10-19
|
@@ -78,7 +78,7 @@ describe PgShrink do
|
|
78
78
|
end
|
79
79
|
end
|
80
80
|
|
81
|
-
describe
|
81
|
+
describe 'when filtering just with a conditions hash' do
|
82
82
|
before(:each) do
|
83
83
|
database.filter_table(:users) do |f|
|
84
84
|
f.filter_by({
|
@@ -88,30 +88,30 @@ describe PgShrink do
|
|
88
88
|
end
|
89
89
|
end
|
90
90
|
|
91
|
-
it
|
91
|
+
it 'Should not call records in batches' do
|
92
92
|
expect(database).not_to receive(:records_in_batches)
|
93
93
|
database.shrink!
|
94
94
|
end
|
95
95
|
|
96
|
-
it
|
96
|
+
it 'Should call delete_records once' do
|
97
97
|
expect(database).to receive(:delete_records).once
|
98
98
|
database.shrink!
|
99
99
|
end
|
100
100
|
|
101
|
-
it
|
101
|
+
it 'Should result in the appropriate records being deleted' do
|
102
102
|
database.shrink!
|
103
103
|
remaining_users = database.connection.from(:users).all
|
104
104
|
expect(remaining_users.size).to eq(10)
|
105
105
|
end
|
106
106
|
|
107
|
-
describe
|
107
|
+
describe 'with a subtable_filter' do
|
108
108
|
before(:each) do
|
109
109
|
database.filter_table(:users) do |f|
|
110
110
|
f.filter_subtable(:user_preferences, :foreign_key => :user_id)
|
111
111
|
end
|
112
112
|
end
|
113
113
|
|
114
|
-
it
|
114
|
+
it 'should remove the appropriate subtable records' do
|
115
115
|
database.shrink!
|
116
116
|
remaining_prefs = database.connection.from(:user_preferences).all
|
117
117
|
expect(remaining_prefs.size).to eq(30)
|
@@ -119,11 +119,11 @@ describe PgShrink do
|
|
119
119
|
end
|
120
120
|
end
|
121
121
|
|
122
|
-
describe
|
122
|
+
describe 'with a test shrinkfile' do
|
123
123
|
let(:shrinkfile) {"spec/Shrinkfile.basic"}
|
124
124
|
let(:url) {database.connection_string}
|
125
125
|
|
126
|
-
it
|
126
|
+
it 'should set up a postgres database' do
|
127
127
|
expect(PgShrink::Database::Postgres).to receive(:new) do |opts|
|
128
128
|
expect(opts[:postgres_url]).to eq(database.connection_string)
|
129
129
|
end.and_return(database)
|
@@ -132,7 +132,7 @@ describe PgShrink do
|
|
132
132
|
end
|
133
133
|
|
134
134
|
|
135
|
-
describe
|
135
|
+
describe 'a simple filter and subtable' do
|
136
136
|
before(:each) do
|
137
137
|
database.filter_table(:users) do |f|
|
138
138
|
f.filter_by "name = 'test 1'"
|
@@ -141,12 +141,12 @@ describe PgShrink do
|
|
141
141
|
database.filter!
|
142
142
|
end
|
143
143
|
|
144
|
-
it
|
144
|
+
it 'will filter users down to the one matching' do
|
145
145
|
remaining_users = database.connection.from(:users).all
|
146
146
|
expect(remaining_users.size).to eq(1)
|
147
147
|
end
|
148
148
|
|
149
|
-
it
|
149
|
+
it 'will filter preferences to only those associated with the user' do
|
150
150
|
remaining_user = database.connection.from(:users).first
|
151
151
|
remaining_preferences = database.connection.
|
152
152
|
from(:user_preferences).all
|
@@ -156,7 +156,7 @@ describe PgShrink do
|
|
156
156
|
end
|
157
157
|
end
|
158
158
|
|
159
|
-
describe
|
159
|
+
describe 'a simple filter and subtable with sanitization on each' do
|
160
160
|
|
161
161
|
before(:each) do
|
162
162
|
database.filter_table(:users) do |f|
|
@@ -179,14 +179,14 @@ describe PgShrink do
|
|
179
179
|
database.shrink!
|
180
180
|
end
|
181
181
|
|
182
|
-
it
|
182
|
+
it 'should result in 1 sanitized users' do
|
183
183
|
remaining_users = database.connection.from(:users).all
|
184
184
|
expect(remaining_users.size).to eq(1)
|
185
185
|
expect(remaining_users.first[:name]).to match(/sanitized/)
|
186
186
|
expect(remaining_users.first[:email]).to match(/blank_email/)
|
187
187
|
end
|
188
188
|
|
189
|
-
it
|
189
|
+
it 'should result in 3 sanitized preferences' do
|
190
190
|
remaining_user = database.connection.from(:users).first
|
191
191
|
remaining_preferences = database.connection.
|
192
192
|
from(:user_preferences).all
|
@@ -197,7 +197,7 @@ describe PgShrink do
|
|
197
197
|
end
|
198
198
|
end
|
199
199
|
end
|
200
|
-
describe
|
200
|
+
describe 'with users and preferences including email as value' do
|
201
201
|
before(:each) do
|
202
202
|
PgSpecHelper.clear_table(database.connection, :users)
|
203
203
|
PgSpecHelper.clear_table(database.connection, :user_preferences)
|
@@ -216,7 +216,7 @@ describe PgShrink do
|
|
216
216
|
end
|
217
217
|
end
|
218
218
|
|
219
|
-
describe
|
219
|
+
describe 'sanitizing subtable' do
|
220
220
|
before(:each) do
|
221
221
|
database.filter_table(:users) do |f|
|
222
222
|
f.sanitize do |u|
|
@@ -227,19 +227,18 @@ describe PgShrink do
|
|
227
227
|
:foreign_key => :user_id,
|
228
228
|
:local_field => :email,
|
229
229
|
:foreign_field => :value,
|
230
|
-
:
|
231
|
-
:type => 'email')
|
230
|
+
:where => { :name => 'email' })
|
232
231
|
end
|
233
232
|
database.shrink!
|
234
233
|
end
|
235
234
|
|
236
|
-
it
|
235
|
+
it 'should sanitize user preference emails' do
|
237
236
|
remaining_preferences = database.connection.
|
238
237
|
from(:user_preferences).where(:name => 'email').all
|
239
238
|
remaining_values = remaining_preferences.map {|p| p[:value]}
|
240
239
|
expect(remaining_values.grep(/blank_email/).size).to eq(20)
|
241
240
|
end
|
242
|
-
it
|
241
|
+
it 'should not sanitize preferences with a different type' do
|
243
242
|
remaining_preferences = database.connection.
|
244
243
|
from(:user_preferences).where(:name => 'name').all
|
245
244
|
remaining_values = remaining_preferences.map {|p| p[:value]}
|
@@ -250,7 +249,7 @@ describe PgShrink do
|
|
250
249
|
end
|
251
250
|
|
252
251
|
|
253
|
-
describe
|
252
|
+
describe 'three table filter chain' do
|
254
253
|
before(:all) do
|
255
254
|
# Rspec doesn't want you using 'let' defined things in before(:all)
|
256
255
|
connection = PgShrink::Database::Postgres.new(PgSpecHelper.pg_config).
|
@@ -260,7 +259,7 @@ describe PgShrink do
|
|
260
259
|
'character varying(256)'})
|
261
260
|
end
|
262
261
|
|
263
|
-
describe
|
262
|
+
describe 'with 20 users and associated preferences' do
|
264
263
|
before(:each) do
|
265
264
|
PgSpecHelper.clear_table(database.connection, :users)
|
266
265
|
PgSpecHelper.clear_table(database.connection, :user_preferences)
|
@@ -288,7 +287,7 @@ describe PgShrink do
|
|
288
287
|
end
|
289
288
|
end
|
290
289
|
|
291
|
-
describe
|
290
|
+
describe 'a simple filter and chained subtables' do
|
292
291
|
before(:each) do
|
293
292
|
database.filter_table(:users) do |f|
|
294
293
|
f.filter_by "name = 'test 1'"
|
@@ -301,11 +300,11 @@ describe PgShrink do
|
|
301
300
|
|
302
301
|
database.filter!
|
303
302
|
end
|
304
|
-
it
|
303
|
+
it 'filters users down to the one matching' do
|
305
304
|
remaining_users = database.connection.from(:users).all
|
306
305
|
expect(remaining_users.size).to eq(1)
|
307
306
|
end
|
308
|
-
it
|
307
|
+
it 'filters preferences to only those associated with the user' do
|
309
308
|
remaining_user = database.connection.from(:users).first
|
310
309
|
remaining_preferences = database.connection.
|
311
310
|
from(:user_preferences).all
|
@@ -313,8 +312,8 @@ describe PgShrink do
|
|
313
312
|
expect(remaining_preferences.map {|u| u[:user_id]}.uniq).
|
314
313
|
to eq([remaining_user[:id]])
|
315
314
|
end
|
316
|
-
it
|
317
|
-
|
315
|
+
it 'filters preference values to those associated with the ' +
|
316
|
+
'preferences remaining' do
|
318
317
|
remaining_user = database.connection.from(:users).first
|
319
318
|
remaining_preferences = database.connection.
|
320
319
|
from(:user_preferences).all
|
@@ -329,7 +328,7 @@ describe PgShrink do
|
|
329
328
|
end
|
330
329
|
end
|
331
330
|
end
|
332
|
-
describe
|
331
|
+
describe 'polymorphic foreign key subtables' do
|
333
332
|
before(:all) do
|
334
333
|
# Rspec doesn't want you using 'let' defined things in before(:all)
|
335
334
|
connection = PgShrink::Database::Postgres.new(PgSpecHelper.pg_config).
|
@@ -343,7 +342,7 @@ describe PgShrink do
|
|
343
342
|
'name' => 'character varying(256)',
|
344
343
|
'value' => 'character varying(256)'})
|
345
344
|
end
|
346
|
-
describe
|
345
|
+
describe 'with 20 users, associated prefs, and prefs for different type' do
|
347
346
|
before(:each) do
|
348
347
|
PgSpecHelper.clear_table(database.connection, :users)
|
349
348
|
PgSpecHelper.clear_table(database.connection, :preferences)
|
@@ -363,17 +362,17 @@ describe PgShrink do
|
|
363
362
|
end
|
364
363
|
end
|
365
364
|
|
366
|
-
describe
|
367
|
-
describe
|
365
|
+
describe 'with condition filters' do
|
366
|
+
describe 'simple cascade' do
|
368
367
|
before(:each) do
|
369
368
|
database.filter_table(:users) do |f|
|
370
|
-
f.filter_by(:
|
371
|
-
f.filter_subtable(:preferences, :
|
372
|
-
|
369
|
+
f.filter_by(name: 'test 1')
|
370
|
+
f.filter_subtable(:preferences, foreign_key: :context_id,
|
371
|
+
where: { context_type: 'User' } )
|
373
372
|
end
|
374
373
|
database.filter!
|
375
374
|
end
|
376
|
-
it
|
375
|
+
it 'will filter prefs with context_type User' do
|
377
376
|
#
|
378
377
|
remaining_user = database.connection.from(:users).first
|
379
378
|
remaining_preferences = database.connection.from(:preferences).
|
@@ -383,13 +382,13 @@ describe PgShrink do
|
|
383
382
|
to eq([remaining_user[:id]])
|
384
383
|
end
|
385
384
|
|
386
|
-
it
|
385
|
+
it 'will not filter preferences without context_type user' do
|
387
386
|
remaining_preferences = database.connection.from(:preferences).
|
388
387
|
where(:context_type => 'OtherClass').all
|
389
388
|
expect(remaining_preferences.size).to eq(20)
|
390
389
|
end
|
391
390
|
end
|
392
|
-
describe
|
391
|
+
describe 'an extra layer of polymorphic subtables' do
|
393
392
|
before(:all) do
|
394
393
|
connection = PgShrink::Database::Postgres.new(PgSpecHelper.pg_config).
|
395
394
|
connection
|
@@ -416,20 +415,19 @@ describe PgShrink do
|
|
416
415
|
end
|
417
416
|
|
418
417
|
database.filter_table(:users) do |f|
|
419
|
-
f.filter_by(:
|
420
|
-
f.filter_subtable(:preferences, :
|
421
|
-
|
418
|
+
f.filter_by(name: 'test 1')
|
419
|
+
f.filter_subtable(:preferences, foreign_key: :context_id,
|
420
|
+
where: { context_type: 'User' } )
|
422
421
|
end
|
423
422
|
|
424
423
|
database.filter_table(:preferences) do |f|
|
425
424
|
f.filter_subtable(:preference_dependents,
|
426
|
-
:
|
427
|
-
|
428
|
-
:type => 'Preference')
|
425
|
+
foreign_key: :context_id,
|
426
|
+
where: { context_type: 'Preference' })
|
429
427
|
end
|
430
428
|
database.filter!
|
431
429
|
end
|
432
|
-
it
|
430
|
+
it 'will filter preference dependents associated with preferences' do
|
433
431
|
remaining_preferences = database.connection.from(:preferences).all
|
434
432
|
remaining_dependents = database.connection.
|
435
433
|
from(:preference_dependents).
|
@@ -438,7 +436,7 @@ describe PgShrink do
|
|
438
436
|
expect(remaining_dependents.size).to eq(remaining_preferences.size)
|
439
437
|
end
|
440
438
|
|
441
|
-
it
|
439
|
+
it 'will not filter preference dependents with different type' do
|
442
440
|
other_dependents = database.connection.
|
443
441
|
from(:preference_dependents).
|
444
442
|
where(:context_type => 'SomeOtherClass').all
|
@@ -447,17 +445,17 @@ describe PgShrink do
|
|
447
445
|
end
|
448
446
|
end
|
449
447
|
|
450
|
-
describe
|
448
|
+
describe 'simple two table filtering' do
|
451
449
|
before(:each) do
|
452
450
|
database.filter_table(:users) do |f|
|
453
451
|
f.filter_by "name = 'test 1'"
|
454
|
-
f.filter_subtable(:preferences, :
|
455
|
-
|
452
|
+
f.filter_subtable(:preferences, foreign_key: :context_id,
|
453
|
+
where: {context_type: 'User'})
|
456
454
|
end
|
457
455
|
database.filter!
|
458
456
|
end
|
459
457
|
|
460
|
-
it
|
458
|
+
it 'will filter prefs with context_type User' do
|
461
459
|
remaining_user = database.connection.from(:users).first
|
462
460
|
remaining_preferences = database.connection.from(:preferences).
|
463
461
|
where(:context_type => 'User').all
|
@@ -466,14 +464,14 @@ describe PgShrink do
|
|
466
464
|
to eq([remaining_user[:id]])
|
467
465
|
end
|
468
466
|
|
469
|
-
it
|
467
|
+
it 'will not filter preferences without context_type user' do
|
470
468
|
remaining_preferences = database.connection.from(:preferences).
|
471
469
|
where(:context_type => 'OtherClass').all
|
472
470
|
expect(remaining_preferences.size).to eq(20)
|
473
471
|
end
|
474
472
|
end
|
475
473
|
|
476
|
-
describe
|
474
|
+
describe 'an extra layer of polymorphic subtables' do
|
477
475
|
before(:all) do
|
478
476
|
connection = PgShrink::Database::Postgres.new(PgSpecHelper.pg_config).
|
479
477
|
connection
|
@@ -501,20 +499,19 @@ describe PgShrink do
|
|
501
499
|
|
502
500
|
database.filter_table(:users) do |f|
|
503
501
|
f.filter_by "name = 'test 1'"
|
504
|
-
f.filter_subtable(:preferences, :
|
505
|
-
|
502
|
+
f.filter_subtable(:preferences, foreign_key: :context_id,
|
503
|
+
where: {context_type: 'User'} )
|
506
504
|
end
|
507
505
|
|
508
506
|
database.filter_table(:preferences) do |f|
|
509
507
|
f.filter_subtable(:preference_dependents,
|
510
|
-
:
|
511
|
-
|
512
|
-
:type => 'Preference')
|
508
|
+
foreign_key: :context_id,
|
509
|
+
where: {context_type: 'Preference'} )
|
513
510
|
end
|
514
511
|
database.filter!
|
515
512
|
end
|
516
513
|
|
517
|
-
it
|
514
|
+
it 'will filter preference dependents associated with preferences' do
|
518
515
|
remaining_preferences = database.connection.from(:preferences).all
|
519
516
|
remaining_dependents = database.connection.
|
520
517
|
from(:preference_dependents).
|
@@ -523,7 +520,7 @@ describe PgShrink do
|
|
523
520
|
expect(remaining_dependents.size).to eq(remaining_preferences.size)
|
524
521
|
end
|
525
522
|
|
526
|
-
it
|
523
|
+
it 'will not filter preference dependents with different type' do
|
527
524
|
other_dependents = database.connection.
|
528
525
|
from(:preference_dependents).
|
529
526
|
where(:context_type => 'SomeOtherClass').all
|
@@ -532,7 +529,7 @@ describe PgShrink do
|
|
532
529
|
end
|
533
530
|
end
|
534
531
|
end
|
535
|
-
describe
|
532
|
+
describe 'has_and_belongs_to_many join tables' do
|
536
533
|
before(:all) do
|
537
534
|
# Rspec doesn't want you using 'let' defined things in before(;all)
|
538
535
|
connection = PgShrink::Database::Postgres.new(PgSpecHelper.pg_config).
|
@@ -547,7 +544,7 @@ describe PgShrink do
|
|
547
544
|
{'name' => 'character varying(256)'})
|
548
545
|
end
|
549
546
|
|
550
|
-
describe
|
547
|
+
describe 'with 5 users, each with 2 apartments, and 1 apartment shared by all 5' do
|
551
548
|
before(:each) do
|
552
549
|
PgSpecHelper.clear_table(database.connection, :users)
|
553
550
|
PgSpecHelper.clear_table(database.connection, :apartments_users)
|
@@ -572,7 +569,7 @@ describe PgShrink do
|
|
572
569
|
end
|
573
570
|
end
|
574
571
|
end
|
575
|
-
describe
|
572
|
+
describe 'With a simple cascading filter' do
|
576
573
|
before(:each) do
|
577
574
|
database.filter_table(:users) do |f|
|
578
575
|
f.filter_by "name = 'test 1'"
|
@@ -586,13 +583,13 @@ describe PgShrink do
|
|
586
583
|
database.shrink!
|
587
584
|
end
|
588
585
|
|
589
|
-
it
|
586
|
+
it 'Should filter down apartments_users' do
|
590
587
|
u = database.connection.from(:users).where(:name => "test 1").first
|
591
588
|
remaining_join_table = database.connection.from(:apartments_users).all
|
592
589
|
expect(remaining_join_table.size).to eq(3)
|
593
590
|
end
|
594
591
|
|
595
|
-
it
|
592
|
+
it 'Should filter apartments as well' do
|
596
593
|
remaining_join_table = database.connection.from(:apartments_users).all
|
597
594
|
remaining_apartments = database.connection.from(:apartments).all
|
598
595
|
expect(remaining_apartments.size).to eq(3)
|
metadata
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_shrink
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kevin Ball
|
8
|
+
- Matt Nemenman
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
date: 2014-
|
12
|
+
date: 2014-07-10 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
13
14
|
- !ruby/object:Gem::Dependency
|
14
15
|
name: pg
|
@@ -195,6 +196,7 @@ dependencies:
|
|
195
196
|
description: pg_shrink makes it simple to shrink and sanitize a PosrgreSQL database
|
196
197
|
email:
|
197
198
|
- kmball11@gmail.com
|
199
|
+
- matt@apartmentlist.com
|
198
200
|
executables:
|
199
201
|
- pg_shrink
|
200
202
|
extensions: []
|
@@ -229,7 +231,7 @@ files:
|
|
229
231
|
- spec/pg_shrink_spec.rb
|
230
232
|
- spec/pg_spec_helper.rb
|
231
233
|
- spec/spec_helper.rb
|
232
|
-
homepage: https://github.com/apartmentlist/
|
234
|
+
homepage: https://github.com/apartmentlist/pg_shrink
|
233
235
|
licenses:
|
234
236
|
- MIT
|
235
237
|
metadata: {}
|