pgslice 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +30 -10
- data/lib/pgslice.rb +31 -19
- data/lib/pgslice/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1e344dce1fde075ab0621622d3f929e2c6677026
|
4
|
+
data.tar.gz: 9fe6e7825852fb37692ff019908dfb82e5cf4095
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a684bbbbc65be40c4a42850f0b54af657371649efef72f9f89337a8fae4ac046bd1fe758f1632ff2fc15c3a3d1e2c6feec7f2eaa70e13d9f815565993ff0e85a
|
7
|
+
data.tar.gz: 6b5e6fe5958f81085c8ec9a3d0d0161e35aa29f616b2db32dca993b520e14178face97dbe43133dc919b750b24a220cfe5819cd1732f6c64307cf81f6ee22363
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# pgslice
|
2
2
|
|
3
|
-
Postgres partitioning as easy as pie. Works great for both new and existing tables, with zero downtime and minimal app changes.
|
3
|
+
Postgres partitioning as easy as pie. Works great for both new and existing tables, with zero downtime and minimal app changes. Archive older data on a rolling basis to keep your database size under control.
|
4
4
|
|
5
5
|
:tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
|
6
6
|
|
@@ -68,7 +68,12 @@ This will give you the `pgslice` command.
|
|
68
68
|
pgslice fill <table> --swapped
|
69
69
|
```
|
70
70
|
|
71
|
-
8.
|
71
|
+
8. Back up the retired table with a tool like [pg_dump](https://www.postgresql.org/docs/current/static/app-pgdump.html) and drop it
|
72
|
+
|
73
|
+
```sql
|
74
|
+
pg_dump -c -Fc -t <table>_retired $PGSLICE_URL > <table>_retired.dump
|
75
|
+
psql -c "DROP <table>_retired" $PGSLICE_URL
|
76
|
+
```
|
72
77
|
|
73
78
|
## Sample Output
|
74
79
|
|
@@ -86,7 +91,7 @@ CREATE TABLE visits_intermediate (LIKE visits INCLUDING ALL);
|
|
86
91
|
CREATE FUNCTION visits_insert_trigger()
|
87
92
|
RETURNS trigger AS $$
|
88
93
|
BEGIN
|
89
|
-
RAISE EXCEPTION '
|
94
|
+
RAISE EXCEPTION 'Create partitions first.';
|
90
95
|
END;
|
91
96
|
$$ LANGUAGE plpgsql;
|
92
97
|
|
@@ -155,18 +160,18 @@ pgslice fill visits
|
|
155
160
|
|
156
161
|
```sql
|
157
162
|
/* 1 of 3 */
|
158
|
-
INSERT INTO visits_intermediate (id, user_id, ip, created_at)
|
159
|
-
SELECT id, user_id, ip, created_at FROM visits
|
163
|
+
INSERT INTO visits_intermediate ("id", "user_id", "ip", "created_at")
|
164
|
+
SELECT "id", "user_id", "ip", "created_at" FROM visits
|
160
165
|
WHERE id > 0 AND id <= 10000 AND created_at >= '2016-08-01'::date AND created_at < '2016-11-01'::date
|
161
166
|
|
162
167
|
/* 2 of 3 */
|
163
|
-
INSERT INTO visits_intermediate (id, user_id, ip, created_at)
|
164
|
-
SELECT id, user_id, ip, created_at FROM visits
|
168
|
+
INSERT INTO visits_intermediate ("id", "user_id", "ip", "created_at")
|
169
|
+
SELECT "id", "user_id", "ip", "created_at" FROM visits
|
165
170
|
WHERE id > 10000 AND id <= 20000 AND created_at >= '2016-08-01'::date AND created_at < '2016-11-01'::date
|
166
171
|
|
167
172
|
/* 3 of 3 */
|
168
|
-
INSERT INTO visits_intermediate (id, user_id, ip, created_at)
|
169
|
-
SELECT id, user_id, ip, created_at FROM visits
|
173
|
+
INSERT INTO visits_intermediate ("id", "user_id", "ip", "created_at")
|
174
|
+
SELECT "id", "user_id", "ip", "created_at" FROM visits
|
170
175
|
WHERE id > 20000 AND id <= 30000 AND created_at >= '2016-08-01'::date AND created_at < '2016-11-01'::date
|
171
176
|
```
|
172
177
|
|
@@ -218,6 +223,21 @@ WHERE
|
|
218
223
|
-- for months, use to_char(NOW() + INTERVAL '3 months', 'YYYYMM')
|
219
224
|
```
|
220
225
|
|
226
|
+
## Archiving Partitions
|
227
|
+
|
228
|
+
Back up and drop older partitions each day or month.
|
229
|
+
|
230
|
+
```sh
|
231
|
+
pg_dump -c -Fc -t <table>_201609 $PGSLICE_URL > <table>_201609.dump
|
232
|
+
psql -c "DROP <table>_201609" $PGSLICE_URL
|
233
|
+
```
|
234
|
+
|
235
|
+
If you use [Amazon S3](https://aws.amazon.com/s3/) for backups, [s3cmd](https://github.com/s3tools/s3cmd) is a nice tool.
|
236
|
+
|
237
|
+
```sh
|
238
|
+
s3cmd put <table>_201609.dump s3://<s3-bucket>/<table>_201609.dump
|
239
|
+
```
|
240
|
+
|
221
241
|
## Additional Commands
|
222
242
|
|
223
243
|
To undo prep (which will delete partitions), use:
|
@@ -268,7 +288,7 @@ You can also use pgslice to reduce the size of a table without partitioning by c
|
|
268
288
|
|
269
289
|
```sh
|
270
290
|
pgslice prep <table> --no-partition
|
271
|
-
pgslice fill <table> --
|
291
|
+
pgslice fill <table> --where "id > 1000" # use any conditions
|
272
292
|
pgslice swap <table>
|
273
293
|
```
|
274
294
|
|
data/lib/pgslice.rb
CHANGED
@@ -79,7 +79,7 @@ CREATE TABLE #{intermediate_table} (LIKE #{table} INCLUDING ALL);
|
|
79
79
|
CREATE FUNCTION #{trigger_name}()
|
80
80
|
RETURNS trigger AS $$
|
81
81
|
BEGIN
|
82
|
-
RAISE EXCEPTION '
|
82
|
+
RAISE EXCEPTION 'Create partitions first.';
|
83
83
|
END;
|
84
84
|
$$ LANGUAGE plpgsql;
|
85
85
|
SQL
|
@@ -133,16 +133,13 @@ SQL
|
|
133
133
|
|
134
134
|
queries = []
|
135
135
|
|
136
|
-
|
137
|
-
|
138
|
-
field, period = comment["comment"].split(",").map { |v| v.split(":").last } rescue [nil, nil]
|
139
|
-
end
|
136
|
+
period, field, needs_comment = settings_from_trigger(original_table, table)
|
137
|
+
abort "Could not read settings" unless period
|
140
138
|
|
141
|
-
|
142
|
-
period, field = settings_from_table(original_table, table)
|
139
|
+
if needs_comment
|
143
140
|
queries << "COMMENT ON TRIGGER #{trigger_name} ON #{table} is 'column:#{field},period:#{period}';"
|
144
141
|
end
|
145
|
-
|
142
|
+
|
146
143
|
# today = utc date
|
147
144
|
today = round_date(DateTime.now.new_offset(0).to_date, period)
|
148
145
|
added_partitions = []
|
@@ -229,7 +226,7 @@ CREATE OR REPLACE FUNCTION #{trigger_name}()
|
|
229
226
|
abort "Table not found: #{source_table}" unless table_exists?(source_table)
|
230
227
|
abort "Table not found: #{dest_table}" unless table_exists?(dest_table)
|
231
228
|
|
232
|
-
period, field =
|
229
|
+
period, field, needs_comment = settings_from_trigger(table, dest_table)
|
233
230
|
|
234
231
|
if period
|
235
232
|
name_format = self.name_format(period)
|
@@ -258,7 +255,7 @@ CREATE OR REPLACE FUNCTION #{trigger_name}()
|
|
258
255
|
end
|
259
256
|
|
260
257
|
starting_id = max_dest_id
|
261
|
-
fields = columns(source_table).join(", ")
|
258
|
+
fields = columns(source_table).map { |c| PG::Connection.quote_ident(c) }.join(", ")
|
262
259
|
batch_size = options[:batch_size]
|
263
260
|
|
264
261
|
i = 1
|
@@ -268,6 +265,9 @@ CREATE OR REPLACE FUNCTION #{trigger_name}()
|
|
268
265
|
if period
|
269
266
|
where << " AND #{field} >= #{sql_date(starting_time)} AND #{field} < #{sql_date(ending_time)}"
|
270
267
|
end
|
268
|
+
if options[:where]
|
269
|
+
where << " AND #{options[:where]}"
|
270
|
+
end
|
271
271
|
|
272
272
|
query = <<-SQL
|
273
273
|
/* #{i} of #{batch_count} */
|
@@ -348,6 +348,7 @@ INSERT INTO #{dest_table} (#{fields})
|
|
348
348
|
o.integer "--start"
|
349
349
|
o.string "--url"
|
350
350
|
o.string "--source-table"
|
351
|
+
o.string "--where"
|
351
352
|
o.on "-v", "--version", "print the version" do
|
352
353
|
log PgSlice::VERSION
|
353
354
|
@exit = true
|
@@ -527,16 +528,27 @@ INSERT INTO #{dest_table} (#{fields})
|
|
527
528
|
end
|
528
529
|
end
|
529
530
|
|
530
|
-
def
|
531
|
+
def settings_from_trigger(original_table, table)
|
531
532
|
trigger_name = self.trigger_name(original_table)
|
532
|
-
|
533
|
-
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
|
539
|
-
|
533
|
+
|
534
|
+
needs_comment = false
|
535
|
+
comment = execute("SELECT obj_description(oid, 'pg_trigger') AS comment FROM pg_trigger WHERE tgname = $1 AND tgrelid = $2::regclass", [trigger_name, table])[0]
|
536
|
+
if comment
|
537
|
+
field, period = comment["comment"].split(",").map { |v| v.split(":").last } rescue [nil, nil]
|
538
|
+
end
|
539
|
+
|
540
|
+
unless period
|
541
|
+
needs_comment = true
|
542
|
+
function_def = execute("select pg_get_functiondef(oid) from pg_proc where proname = $1", [trigger_name])[0]
|
543
|
+
return [nil, nil] unless function_def
|
544
|
+
function_def = function_def["pg_get_functiondef"]
|
545
|
+
sql_format = SQL_FORMAT.find { |_, f| function_def.include?("'#{f}'") }
|
546
|
+
return [nil, nil] unless sql_format
|
547
|
+
period = sql_format[0]
|
548
|
+
field = /to_char\(NEW\.(\w+),/.match(function_def)[1]
|
549
|
+
end
|
550
|
+
|
551
|
+
[period, field, needs_comment]
|
540
552
|
end
|
541
553
|
end
|
542
554
|
end
|
data/lib/pgslice/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pgslice
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Kane
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-09-
|
11
|
+
date: 2016-09-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: slop
|