pgslice 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/README.md +14 -0
- data/exe/pgslice +2 -4
- data/lib/pgslice.rb +8 -742
- data/lib/pgslice/client.rb +577 -0
- data/lib/pgslice/generic_table.rb +89 -0
- data/lib/pgslice/table.rb +72 -0
- data/lib/pgslice/version.rb +1 -1
- metadata +10 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d02cf8d6926bc4ed8908f1c9cc06bf406a3a926d99c08cba1e23f1ef8279f8fc
|
4
|
+
data.tar.gz: 4d68a0f7e771714d488f5e1c0bb4410f342f9f2e7e561fece795ee8d90b0e479
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 419d9ca61bc0fd990d5680fbb35c9b8a2de88b169b2882c5b28a8233082bf5f2c75e0cb9c531fb9538d0815777500ea773e2a61fff4d93b1e481ae81dc2d48a2
|
7
|
+
data.tar.gz: 55555b66db960941e2d766af939637d10b6fac19301ee5b3eb141ccb860963fdbdfd98bb2fac0f459184f75b084d1d463e095d644efc16161dae18285b36f66a
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -365,6 +365,20 @@ Once a table is partitioned, here’s how to change the schema:
|
|
365
365
|
|
366
366
|
Postgres 10 introduces [declarative partitioning](https://www.postgresql.org/docs/10/static/ddl-partitioning.html#ddl-partitioning-declarative). A major benefit is `INSERT` statements with a `RETURNING` clause work as expected. If you prefer to use trigger-based partitioning instead (not recommended), pass the `--trigger-based` option to the `prep` command.
|
367
367
|
|
368
|
+
## Data Protection
|
369
|
+
|
370
|
+
When connecting to a remote database, make sure your connection is secure.
|
371
|
+
|
372
|
+
If you do not use a VPN, you must use `sslmode=verify-full` with a root certificate to [protect against MITM attacks](https://www.postgresql.org/docs/current/static/libpq-ssl.html). If you don’t do this, your database credentials can be compromised. This cannot be understated!
|
373
|
+
|
374
|
+
Surprisingly and unfortunately, there’s [not a secure way](https://thusoy.com/2016/mitming-postgres) to connect to Heroku Postgres with any client.
|
375
|
+
|
376
|
+
For Amazon RDS, download the [root certificate](https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem) and append to your database url:
|
377
|
+
|
378
|
+
```
|
379
|
+
?sslmode=verify-full&sslrootcert=rds-combined-ca-bundle.pem
|
380
|
+
```
|
381
|
+
|
368
382
|
## Upgrading
|
369
383
|
|
370
384
|
Run:
|
data/exe/pgslice
CHANGED
data/lib/pgslice.rb
CHANGED
@@ -1,744 +1,10 @@
|
|
1
|
-
|
2
|
-
require "slop"
|
3
|
-
require "pg"
|
1
|
+
# dependencies
|
4
2
|
require "cgi"
|
3
|
+
require "thor"
|
4
|
+
require "pg"
|
5
5
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
SQL_FORMAT = {
|
13
|
-
day: "YYYYMMDD",
|
14
|
-
month: "YYYYMM"
|
15
|
-
}
|
16
|
-
|
17
|
-
def initialize(args)
|
18
|
-
$stdout.sync = true
|
19
|
-
$stderr.sync = true
|
20
|
-
parse_args(args)
|
21
|
-
@command = @arguments.shift
|
22
|
-
end
|
23
|
-
|
24
|
-
def perform
|
25
|
-
return if @exit
|
26
|
-
|
27
|
-
case @command
|
28
|
-
when "prep"
|
29
|
-
prep
|
30
|
-
when "add_partitions"
|
31
|
-
add_partitions
|
32
|
-
when "fill"
|
33
|
-
fill
|
34
|
-
when "swap"
|
35
|
-
swap
|
36
|
-
when "unswap"
|
37
|
-
unswap
|
38
|
-
when "unprep"
|
39
|
-
unprep
|
40
|
-
when "analyze"
|
41
|
-
analyze
|
42
|
-
when nil
|
43
|
-
log "Commands: add_partitions, analyze, fill, prep, swap, unprep, unswap"
|
44
|
-
else
|
45
|
-
abort "Unknown command: #{@command}"
|
46
|
-
end
|
47
|
-
ensure
|
48
|
-
@connection.close if @connection
|
49
|
-
end
|
50
|
-
|
51
|
-
protected
|
52
|
-
|
53
|
-
# commands
|
54
|
-
|
55
|
-
def prep
|
56
|
-
table, column, period = arguments
|
57
|
-
table = qualify_table(table)
|
58
|
-
intermediate_table = "#{table}_intermediate"
|
59
|
-
|
60
|
-
trigger_name = self.trigger_name(table)
|
61
|
-
|
62
|
-
if options[:no_partition]
|
63
|
-
abort "Usage: pgslice prep <table> --no-partition" if arguments.length != 1
|
64
|
-
abort "Can't use --trigger-based and --no-partition" if options[:trigger_based]
|
65
|
-
else
|
66
|
-
abort "Usage: pgslice prep <table> <column> <period>" if arguments.length != 3
|
67
|
-
end
|
68
|
-
abort "Table not found: #{table}" unless table_exists?(table)
|
69
|
-
abort "Table already exists: #{intermediate_table}" if table_exists?(intermediate_table)
|
70
|
-
|
71
|
-
unless options[:no_partition]
|
72
|
-
abort "Column not found: #{column}" unless columns(table).include?(column)
|
73
|
-
abort "Invalid period: #{period}" unless SQL_FORMAT[period.to_sym]
|
74
|
-
end
|
75
|
-
|
76
|
-
queries = []
|
77
|
-
|
78
|
-
declarative = server_version_num >= 100000 && !options[:trigger_based]
|
79
|
-
|
80
|
-
if declarative && !options[:no_partition]
|
81
|
-
queries << <<-SQL
|
82
|
-
CREATE TABLE #{quote_table(intermediate_table)} (LIKE #{quote_table(table)} INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING STORAGE INCLUDING COMMENTS) PARTITION BY RANGE (#{quote_table(column)});
|
83
|
-
SQL
|
84
|
-
|
85
|
-
if server_version_num >= 110000
|
86
|
-
index_defs = execute("SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indrelid = #{regclass(table)} AND indisprimary = 'f'").map { |r| r["pg_get_indexdef"] }
|
87
|
-
index_defs.each do |index_def|
|
88
|
-
queries << index_def.sub(/ ON \S+ USING /, " ON #{quote_table(intermediate_table)} USING ").sub(/ INDEX .+ ON /, " INDEX ON ") + ";"
|
89
|
-
end
|
90
|
-
end
|
91
|
-
|
92
|
-
# add comment
|
93
|
-
cast = column_cast(table, column)
|
94
|
-
queries << <<-SQL
|
95
|
-
COMMENT ON TABLE #{quote_table(intermediate_table)} is 'column:#{column},period:#{period},cast:#{cast}';
|
96
|
-
SQL
|
97
|
-
else
|
98
|
-
queries << <<-SQL
|
99
|
-
CREATE TABLE #{quote_table(intermediate_table)} (LIKE #{quote_table(table)} INCLUDING ALL);
|
100
|
-
SQL
|
101
|
-
|
102
|
-
foreign_keys(table).each do |fk_def|
|
103
|
-
queries << "ALTER TABLE #{quote_table(intermediate_table)} ADD #{fk_def};"
|
104
|
-
end
|
105
|
-
end
|
106
|
-
|
107
|
-
if !options[:no_partition] && !declarative
|
108
|
-
sql_format = SQL_FORMAT[period.to_sym]
|
109
|
-
queries << <<-SQL
|
110
|
-
CREATE FUNCTION #{quote_ident(trigger_name)}()
|
111
|
-
RETURNS trigger AS $$
|
112
|
-
BEGIN
|
113
|
-
RAISE EXCEPTION 'Create partitions first.';
|
114
|
-
END;
|
115
|
-
$$ LANGUAGE plpgsql;
|
116
|
-
SQL
|
117
|
-
|
118
|
-
queries << <<-SQL
|
119
|
-
CREATE TRIGGER #{quote_ident(trigger_name)}
|
120
|
-
BEFORE INSERT ON #{quote_table(intermediate_table)}
|
121
|
-
FOR EACH ROW EXECUTE PROCEDURE #{quote_ident(trigger_name)}();
|
122
|
-
SQL
|
123
|
-
|
124
|
-
cast = column_cast(table, column)
|
125
|
-
queries << <<-SQL
|
126
|
-
COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_table(intermediate_table)} is 'column:#{column},period:#{period},cast:#{cast}';
|
127
|
-
SQL
|
128
|
-
end
|
129
|
-
|
130
|
-
run_queries(queries)
|
131
|
-
end
|
132
|
-
|
133
|
-
def unprep
|
134
|
-
table = qualify_table(arguments.first)
|
135
|
-
intermediate_table = "#{table}_intermediate"
|
136
|
-
trigger_name = self.trigger_name(table)
|
137
|
-
|
138
|
-
abort "Usage: pgslice unprep <table>" if arguments.length != 1
|
139
|
-
abort "Table not found: #{intermediate_table}" unless table_exists?(intermediate_table)
|
140
|
-
|
141
|
-
queries = [
|
142
|
-
"DROP TABLE #{quote_table(intermediate_table)} CASCADE;",
|
143
|
-
"DROP FUNCTION IF EXISTS #{quote_ident(trigger_name)}();"
|
144
|
-
]
|
145
|
-
run_queries(queries)
|
146
|
-
end
|
147
|
-
|
148
|
-
def add_partitions
|
149
|
-
original_table = qualify_table(arguments.first)
|
150
|
-
table = options[:intermediate] ? "#{original_table}_intermediate" : original_table
|
151
|
-
trigger_name = self.trigger_name(original_table)
|
152
|
-
|
153
|
-
abort "Usage: pgslice add_partitions <table>" if arguments.length != 1
|
154
|
-
abort "Table not found: #{table}" unless table_exists?(table)
|
155
|
-
|
156
|
-
future = options[:future]
|
157
|
-
past = options[:past]
|
158
|
-
range = (-1 * past)..future
|
159
|
-
|
160
|
-
period, field, cast, needs_comment, declarative = settings_from_trigger(original_table, table)
|
161
|
-
unless period
|
162
|
-
message = "No settings found: #{table}"
|
163
|
-
message = "#{message}\nDid you mean to use --intermediate?" unless options[:intermediate]
|
164
|
-
abort message
|
165
|
-
end
|
166
|
-
|
167
|
-
queries = []
|
168
|
-
|
169
|
-
if needs_comment
|
170
|
-
queries << "COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_table(table)} is 'column:#{field},period:#{period},cast:#{cast}';"
|
171
|
-
end
|
172
|
-
|
173
|
-
# today = utc date
|
174
|
-
today = round_date(DateTime.now.new_offset(0).to_date, period)
|
175
|
-
|
176
|
-
schema_table =
|
177
|
-
if !declarative
|
178
|
-
table
|
179
|
-
elsif options[:intermediate]
|
180
|
-
original_table
|
181
|
-
else
|
182
|
-
existing_partitions(original_table, period).last
|
183
|
-
end
|
184
|
-
|
185
|
-
# indexes automatically propagate in Postgres 11+
|
186
|
-
index_defs =
|
187
|
-
if !declarative || server_version_num < 110000
|
188
|
-
execute("SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indrelid = #{regclass(schema_table)} AND indisprimary = 'f'").map { |r| r["pg_get_indexdef"] }
|
189
|
-
else
|
190
|
-
[]
|
191
|
-
end
|
192
|
-
|
193
|
-
fk_defs = foreign_keys(schema_table)
|
194
|
-
primary_key = self.primary_key(schema_table)
|
195
|
-
|
196
|
-
added_partitions = []
|
197
|
-
range.each do |n|
|
198
|
-
day = advance_date(today, period, n)
|
199
|
-
|
200
|
-
partition_name = "#{original_table}_#{day.strftime(name_format(period))}"
|
201
|
-
next if table_exists?(partition_name)
|
202
|
-
added_partitions << partition_name
|
203
|
-
|
204
|
-
if declarative
|
205
|
-
queries << <<-SQL
|
206
|
-
CREATE TABLE #{quote_table(partition_name)} PARTITION OF #{quote_table(table)} FOR VALUES FROM (#{sql_date(day, cast, false)}) TO (#{sql_date(advance_date(day, period, 1), cast, false)});
|
207
|
-
SQL
|
208
|
-
else
|
209
|
-
queries << <<-SQL
|
210
|
-
CREATE TABLE #{quote_table(partition_name)}
|
211
|
-
(CHECK (#{quote_ident(field)} >= #{sql_date(day, cast)} AND #{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}))
|
212
|
-
INHERITS (#{quote_table(table)});
|
213
|
-
SQL
|
214
|
-
end
|
215
|
-
|
216
|
-
queries << "ALTER TABLE #{quote_table(partition_name)} ADD PRIMARY KEY (#{primary_key.map { |k| quote_ident(k) }.join(", ")});" if primary_key.any?
|
217
|
-
|
218
|
-
index_defs.each do |index_def|
|
219
|
-
queries << index_def.sub(/ ON \S+ USING /, " ON #{quote_table(partition_name)} USING ").sub(/ INDEX .+ ON /, " INDEX ON ") + ";"
|
220
|
-
end
|
221
|
-
|
222
|
-
fk_defs.each do |fk_def|
|
223
|
-
queries << "ALTER TABLE #{quote_table(partition_name)} ADD #{fk_def};"
|
224
|
-
end
|
225
|
-
end
|
226
|
-
|
227
|
-
unless declarative
|
228
|
-
# update trigger based on existing partitions
|
229
|
-
current_defs = []
|
230
|
-
future_defs = []
|
231
|
-
past_defs = []
|
232
|
-
name_format = self.name_format(period)
|
233
|
-
existing_tables = existing_partitions(original_table, period)
|
234
|
-
existing_tables = (existing_tables + added_partitions).uniq.sort
|
235
|
-
|
236
|
-
existing_tables.each do |table|
|
237
|
-
day = DateTime.strptime(table.split("_").last, name_format)
|
238
|
-
partition_name = "#{original_table}_#{day.strftime(name_format(period))}"
|
239
|
-
|
240
|
-
sql = "(NEW.#{quote_ident(field)} >= #{sql_date(day, cast)} AND NEW.#{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}) THEN
|
241
|
-
INSERT INTO #{quote_table(partition_name)} VALUES (NEW.*);"
|
242
|
-
|
243
|
-
if day.to_date < today
|
244
|
-
past_defs << sql
|
245
|
-
elsif advance_date(day, period, 1) < today
|
246
|
-
current_defs << sql
|
247
|
-
else
|
248
|
-
future_defs << sql
|
249
|
-
end
|
250
|
-
end
|
251
|
-
|
252
|
-
# order by current period, future periods asc, past periods desc
|
253
|
-
trigger_defs = current_defs + future_defs + past_defs.reverse
|
254
|
-
|
255
|
-
if trigger_defs.any?
|
256
|
-
queries << <<-SQL
|
257
|
-
CREATE OR REPLACE FUNCTION #{quote_ident(trigger_name)}()
|
258
|
-
RETURNS trigger AS $$
|
259
|
-
BEGIN
|
260
|
-
IF #{trigger_defs.join("\n ELSIF ")}
|
261
|
-
ELSE
|
262
|
-
RAISE EXCEPTION 'Date out of range. Ensure partitions are created.';
|
263
|
-
END IF;
|
264
|
-
RETURN NULL;
|
265
|
-
END;
|
266
|
-
$$ LANGUAGE plpgsql;
|
267
|
-
SQL
|
268
|
-
end
|
269
|
-
end
|
270
|
-
|
271
|
-
run_queries(queries) if queries.any?
|
272
|
-
end
|
273
|
-
|
274
|
-
def fill
|
275
|
-
table = qualify_table(arguments.first)
|
276
|
-
|
277
|
-
abort "Usage: pgslice fill <table>" if arguments.length != 1
|
278
|
-
|
279
|
-
source_table = options[:source_table]
|
280
|
-
dest_table = options[:dest_table]
|
281
|
-
|
282
|
-
if options[:swapped]
|
283
|
-
source_table ||= retired_name(table)
|
284
|
-
dest_table ||= table
|
285
|
-
else
|
286
|
-
source_table ||= table
|
287
|
-
dest_table ||= intermediate_name(table)
|
288
|
-
end
|
289
|
-
|
290
|
-
abort "Table not found: #{source_table}" unless table_exists?(source_table)
|
291
|
-
abort "Table not found: #{dest_table}" unless table_exists?(dest_table)
|
292
|
-
|
293
|
-
period, field, cast, needs_comment, declarative = settings_from_trigger(table, dest_table)
|
294
|
-
|
295
|
-
if period
|
296
|
-
name_format = self.name_format(period)
|
297
|
-
|
298
|
-
existing_tables = existing_partitions(table, period)
|
299
|
-
if existing_tables.any?
|
300
|
-
starting_time = DateTime.strptime(existing_tables.first.split("_").last, name_format)
|
301
|
-
ending_time = advance_date(DateTime.strptime(existing_tables.last.split("_").last, name_format), period, 1)
|
302
|
-
end
|
303
|
-
end
|
304
|
-
|
305
|
-
schema_table = period && declarative ? existing_tables.last : table
|
306
|
-
|
307
|
-
primary_key = self.primary_key(schema_table)[0]
|
308
|
-
abort "No primary key" unless primary_key
|
309
|
-
|
310
|
-
max_source_id = nil
|
311
|
-
begin
|
312
|
-
max_source_id = max_id(source_table, primary_key)
|
313
|
-
rescue PG::UndefinedFunction
|
314
|
-
abort "Only numeric primary keys are supported"
|
315
|
-
end
|
316
|
-
|
317
|
-
max_dest_id =
|
318
|
-
if options[:start]
|
319
|
-
options[:start]
|
320
|
-
elsif options[:swapped]
|
321
|
-
max_id(dest_table, primary_key, where: options[:where], below: max_source_id)
|
322
|
-
else
|
323
|
-
max_id(dest_table, primary_key, where: options[:where])
|
324
|
-
end
|
325
|
-
|
326
|
-
if max_dest_id == 0 && !options[:swapped]
|
327
|
-
min_source_id = min_id(source_table, primary_key, field, cast, starting_time, options[:where])
|
328
|
-
max_dest_id = min_source_id - 1 if min_source_id
|
329
|
-
end
|
330
|
-
|
331
|
-
starting_id = max_dest_id
|
332
|
-
fields = columns(source_table).map { |c| quote_ident(c) }.join(", ")
|
333
|
-
batch_size = options[:batch_size]
|
334
|
-
|
335
|
-
i = 1
|
336
|
-
batch_count = ((max_source_id - starting_id) / batch_size.to_f).ceil
|
337
|
-
|
338
|
-
if batch_count == 0
|
339
|
-
log_sql "/* nothing to fill */"
|
340
|
-
end
|
341
|
-
|
342
|
-
while starting_id < max_source_id
|
343
|
-
where = "#{quote_ident(primary_key)} > #{starting_id} AND #{quote_ident(primary_key)} <= #{starting_id + batch_size}"
|
344
|
-
if starting_time
|
345
|
-
where << " AND #{quote_ident(field)} >= #{sql_date(starting_time, cast)} AND #{quote_ident(field)} < #{sql_date(ending_time, cast)}"
|
346
|
-
end
|
347
|
-
if options[:where]
|
348
|
-
where << " AND #{options[:where]}"
|
349
|
-
end
|
350
|
-
|
351
|
-
query = <<-SQL
|
352
|
-
/* #{i} of #{batch_count} */
|
353
|
-
INSERT INTO #{quote_table(dest_table)} (#{fields})
|
354
|
-
SELECT #{fields} FROM #{quote_table(source_table)}
|
355
|
-
WHERE #{where}
|
356
|
-
SQL
|
357
|
-
|
358
|
-
run_query(query)
|
359
|
-
|
360
|
-
starting_id += batch_size
|
361
|
-
i += 1
|
362
|
-
|
363
|
-
if options[:sleep] && starting_id <= max_source_id
|
364
|
-
sleep(options[:sleep])
|
365
|
-
end
|
366
|
-
end
|
367
|
-
end
|
368
|
-
|
369
|
-
def swap
|
370
|
-
table = qualify_table(arguments.first)
|
371
|
-
intermediate_table = intermediate_name(table)
|
372
|
-
retired_table = retired_name(table)
|
373
|
-
|
374
|
-
abort "Usage: pgslice swap <table>" if arguments.length != 1
|
375
|
-
abort "Table not found: #{table}" unless table_exists?(table)
|
376
|
-
abort "Table not found: #{intermediate_table}" unless table_exists?(intermediate_table)
|
377
|
-
abort "Table already exists: #{retired_table}" if table_exists?(retired_table)
|
378
|
-
|
379
|
-
queries = [
|
380
|
-
"ALTER TABLE #{quote_table(table)} RENAME TO #{quote_no_schema(retired_table)};",
|
381
|
-
"ALTER TABLE #{quote_table(intermediate_table)} RENAME TO #{quote_no_schema(table)};"
|
382
|
-
]
|
383
|
-
|
384
|
-
self.sequences(table).each do |sequence|
|
385
|
-
queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_name"])} OWNED BY #{quote_ident(table)}.#{quote_ident(sequence["related_column"])};"
|
386
|
-
end
|
387
|
-
|
388
|
-
queries.unshift("SET LOCAL lock_timeout = '#{options[:lock_timeout]}';") if server_version_num >= 90300
|
389
|
-
|
390
|
-
run_queries(queries)
|
391
|
-
end
|
392
|
-
|
393
|
-
def unswap
|
394
|
-
table = qualify_table(arguments.first)
|
395
|
-
intermediate_table = intermediate_name(table)
|
396
|
-
retired_table = retired_name(table)
|
397
|
-
|
398
|
-
abort "Usage: pgslice unswap <table>" if arguments.length != 1
|
399
|
-
abort "Table not found: #{table}" unless table_exists?(table)
|
400
|
-
abort "Table not found: #{retired_table}" unless table_exists?(retired_table)
|
401
|
-
abort "Table already exists: #{intermediate_table}" if table_exists?(intermediate_table)
|
402
|
-
|
403
|
-
queries = [
|
404
|
-
"ALTER TABLE #{quote_table(table)} RENAME TO #{quote_no_schema(intermediate_table)};",
|
405
|
-
"ALTER TABLE #{quote_table(retired_table)} RENAME TO #{quote_no_schema(table)};"
|
406
|
-
]
|
407
|
-
|
408
|
-
self.sequences(table).each do |sequence|
|
409
|
-
queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_name"])} OWNED BY #{quote_ident(table)}.#{quote_ident(sequence["related_column"])};"
|
410
|
-
end
|
411
|
-
|
412
|
-
run_queries(queries)
|
413
|
-
end
|
414
|
-
|
415
|
-
def analyze
|
416
|
-
table = qualify_table(arguments.first)
|
417
|
-
parent_table = options[:swapped] ? table : intermediate_name(table)
|
418
|
-
|
419
|
-
abort "Usage: pgslice analyze <table>" if arguments.length != 1
|
420
|
-
|
421
|
-
existing_tables = existing_partitions(table)
|
422
|
-
analyze_list = existing_tables + [parent_table]
|
423
|
-
run_queries_without_transaction analyze_list.map { |t| "ANALYZE VERBOSE #{quote_table(t)};" }
|
424
|
-
end
|
425
|
-
|
426
|
-
# arguments
|
427
|
-
|
428
|
-
def parse_args(args)
|
429
|
-
opts = Slop.parse(args) do |o|
|
430
|
-
o.boolean "--intermediate"
|
431
|
-
o.boolean "--swapped"
|
432
|
-
o.float "--sleep"
|
433
|
-
o.integer "--future", default: 0
|
434
|
-
o.integer "--past", default: 0
|
435
|
-
o.integer "--batch-size", default: 10000
|
436
|
-
o.boolean "--dry-run", default: false
|
437
|
-
o.boolean "--no-partition", default: false
|
438
|
-
o.boolean "--trigger-based", default: false
|
439
|
-
o.integer "--start"
|
440
|
-
o.string "--url"
|
441
|
-
o.string "--source-table"
|
442
|
-
o.string "--dest-table"
|
443
|
-
o.string "--where"
|
444
|
-
o.string "--lock-timeout", default: "5s"
|
445
|
-
o.on "-v", "--version", "print the version" do
|
446
|
-
log PgSlice::VERSION
|
447
|
-
@exit = true
|
448
|
-
end
|
449
|
-
end
|
450
|
-
@arguments = opts.arguments
|
451
|
-
@options = opts.to_hash
|
452
|
-
rescue Slop::Error => e
|
453
|
-
abort e.message
|
454
|
-
end
|
455
|
-
|
456
|
-
# output
|
457
|
-
|
458
|
-
def log(message = nil)
|
459
|
-
$stderr.puts message
|
460
|
-
end
|
461
|
-
|
462
|
-
def log_sql(message = nil)
|
463
|
-
$stdout.puts message
|
464
|
-
end
|
465
|
-
|
466
|
-
def abort(message)
|
467
|
-
raise PgSlice::Error, message
|
468
|
-
end
|
469
|
-
|
470
|
-
# database connection
|
471
|
-
|
472
|
-
def connection
|
473
|
-
@connection ||= begin
|
474
|
-
url = options[:url] || ENV["PGSLICE_URL"]
|
475
|
-
abort "Set PGSLICE_URL or use the --url option" unless url
|
476
|
-
|
477
|
-
uri = URI.parse(url)
|
478
|
-
params = CGI.parse(uri.query.to_s)
|
479
|
-
# remove schema
|
480
|
-
@schema = Array(params.delete("schema") || "public")[0]
|
481
|
-
uri.query = URI.encode_www_form(params)
|
482
|
-
|
483
|
-
ENV["PGCONNECT_TIMEOUT"] ||= "1"
|
484
|
-
PG::Connection.new(uri.to_s)
|
485
|
-
end
|
486
|
-
rescue PG::ConnectionBad => e
|
487
|
-
abort e.message
|
488
|
-
rescue URI::InvalidURIError
|
489
|
-
abort "Invalid url"
|
490
|
-
end
|
491
|
-
|
492
|
-
def schema
|
493
|
-
connection # ensure called first
|
494
|
-
@schema
|
495
|
-
end
|
496
|
-
|
497
|
-
def execute(query, params = [])
|
498
|
-
connection.exec_params(query, params).to_a
|
499
|
-
end
|
500
|
-
|
501
|
-
def run_queries(queries)
|
502
|
-
connection.transaction do
|
503
|
-
execute("SET LOCAL client_min_messages TO warning") unless options[:dry_run]
|
504
|
-
log_sql "BEGIN;"
|
505
|
-
log_sql
|
506
|
-
run_queries_without_transaction(queries)
|
507
|
-
log_sql "COMMIT;"
|
508
|
-
end
|
509
|
-
end
|
510
|
-
|
511
|
-
def run_query(query)
|
512
|
-
log_sql query
|
513
|
-
unless options[:dry_run]
|
514
|
-
begin
|
515
|
-
execute(query)
|
516
|
-
rescue PG::ServerError => e
|
517
|
-
abort("#{e.class.name}: #{e.message}")
|
518
|
-
end
|
519
|
-
end
|
520
|
-
log_sql
|
521
|
-
end
|
522
|
-
|
523
|
-
def run_queries_without_transaction(queries)
|
524
|
-
queries.each do |query|
|
525
|
-
run_query(query)
|
526
|
-
end
|
527
|
-
end
|
528
|
-
|
529
|
-
def server_version_num
|
530
|
-
execute("SHOW server_version_num")[0]["server_version_num"].to_i
|
531
|
-
end
|
532
|
-
|
533
|
-
def existing_partitions(table, period = nil)
|
534
|
-
count =
|
535
|
-
case period
|
536
|
-
when "day"
|
537
|
-
8
|
538
|
-
when "month"
|
539
|
-
6
|
540
|
-
else
|
541
|
-
"6,8"
|
542
|
-
end
|
543
|
-
|
544
|
-
existing_tables(like: "#{table}_%").select { |t| /\A#{Regexp.escape("#{table}_")}\d{#{count}}\z/.match(t) }
|
545
|
-
end
|
546
|
-
|
547
|
-
def existing_tables(like:)
|
548
|
-
query = "SELECT schemaname, tablename FROM pg_catalog.pg_tables WHERE schemaname = $1 AND tablename LIKE $2"
|
549
|
-
execute(query, like.split(".", 2)).map { |r| "#{r["schemaname"]}.#{r["tablename"]}" }.sort
|
550
|
-
end
|
551
|
-
|
552
|
-
def table_exists?(table)
|
553
|
-
existing_tables(like: table).any?
|
554
|
-
end
|
555
|
-
|
556
|
-
def columns(table)
|
557
|
-
execute("SELECT column_name FROM information_schema.columns WHERE table_schema || '.' || table_name = $1", [table]).map{ |r| r["column_name"] }
|
558
|
-
end
|
559
|
-
|
560
|
-
# http://stackoverflow.com/a/20537829
|
561
|
-
def primary_key(table)
|
562
|
-
query = <<-SQL
|
563
|
-
SELECT
|
564
|
-
pg_attribute.attname,
|
565
|
-
format_type(pg_attribute.atttypid, pg_attribute.atttypmod)
|
566
|
-
FROM
|
567
|
-
pg_index, pg_class, pg_attribute, pg_namespace
|
568
|
-
WHERE
|
569
|
-
nspname || '.' || relname = $1 AND
|
570
|
-
indrelid = pg_class.oid AND
|
571
|
-
pg_class.relnamespace = pg_namespace.oid AND
|
572
|
-
pg_attribute.attrelid = pg_class.oid AND
|
573
|
-
pg_attribute.attnum = any(pg_index.indkey) AND
|
574
|
-
indisprimary
|
575
|
-
SQL
|
576
|
-
execute(query, [table]).map { |r| r["attname"] }
|
577
|
-
end
|
578
|
-
|
579
|
-
def max_id(table, primary_key, below: nil, where: nil)
|
580
|
-
query = "SELECT MAX(#{quote_ident(primary_key)}) FROM #{quote_table(table)}"
|
581
|
-
conditions = []
|
582
|
-
conditions << "#{quote_ident(primary_key)} <= #{below}" if below
|
583
|
-
conditions << where if where
|
584
|
-
query << " WHERE #{conditions.join(" AND ")}" if conditions.any?
|
585
|
-
execute(query)[0]["max"].to_i
|
586
|
-
end
|
587
|
-
|
588
|
-
def min_id(table, primary_key, column, cast, starting_time, where)
|
589
|
-
query = "SELECT MIN(#{quote_ident(primary_key)}) FROM #{quote_table(table)}"
|
590
|
-
conditions = []
|
591
|
-
conditions << "#{quote_ident(column)} >= #{sql_date(starting_time, cast)}" if starting_time
|
592
|
-
conditions << where if where
|
593
|
-
query << " WHERE #{conditions.join(" AND ")}" if conditions.any?
|
594
|
-
(execute(query)[0]["min"] || 1).to_i
|
595
|
-
end
|
596
|
-
|
597
|
-
def has_trigger?(trigger_name, table)
|
598
|
-
!fetch_trigger(trigger_name, table).nil?
|
599
|
-
end
|
600
|
-
|
601
|
-
def fetch_comment(table)
|
602
|
-
execute("SELECT obj_description(#{regclass(table)}) AS comment")[0]
|
603
|
-
end
|
604
|
-
|
605
|
-
# http://www.dbforums.com/showthread.php?1667561-How-to-list-sequences-and-the-columns-by-SQL
|
606
|
-
def sequences(table)
|
607
|
-
query = <<-SQL
|
608
|
-
SELECT
|
609
|
-
a.attname as related_column,
|
610
|
-
s.relname as sequence_name
|
611
|
-
FROM pg_class s
|
612
|
-
JOIN pg_depend d ON d.objid = s.oid
|
613
|
-
JOIN pg_class t ON d.objid = s.oid AND d.refobjid = t.oid
|
614
|
-
JOIN pg_attribute a ON (d.refobjid, d.refobjsubid) = (a.attrelid, a.attnum)
|
615
|
-
JOIN pg_namespace n ON n.oid = s.relnamespace
|
616
|
-
WHERE s.relkind = 'S'
|
617
|
-
AND n.nspname = $1
|
618
|
-
AND t.relname = $2
|
619
|
-
SQL
|
620
|
-
execute(query, [schema, table])
|
621
|
-
end
|
622
|
-
|
623
|
-
# helpers
|
624
|
-
|
625
|
-
def trigger_name(table)
|
626
|
-
"#{table.split(".")[-1]}_insert_trigger"
|
627
|
-
end
|
628
|
-
|
629
|
-
def intermediate_name(table)
|
630
|
-
"#{table}_intermediate"
|
631
|
-
end
|
632
|
-
|
633
|
-
def retired_name(table)
|
634
|
-
"#{table}_retired"
|
635
|
-
end
|
636
|
-
|
637
|
-
def column_cast(table, column)
|
638
|
-
data_type = execute("SELECT data_type FROM information_schema.columns WHERE table_schema || '.' || table_name = $1 AND column_name = $2", [table, column])[0]["data_type"]
|
639
|
-
data_type == "timestamp with time zone" ? "timestamptz" : "date"
|
640
|
-
end
|
641
|
-
|
642
|
-
def sql_date(time, cast, add_cast = true)
|
643
|
-
if cast == "timestamptz"
|
644
|
-
fmt = "%Y-%m-%d %H:%M:%S UTC"
|
645
|
-
else
|
646
|
-
fmt = "%Y-%m-%d"
|
647
|
-
end
|
648
|
-
str = "'#{time.strftime(fmt)}'"
|
649
|
-
add_cast ? "#{str}::#{cast}" : str
|
650
|
-
end
|
651
|
-
|
652
|
-
def name_format(period)
|
653
|
-
case period.to_sym
|
654
|
-
when :day
|
655
|
-
"%Y%m%d"
|
656
|
-
else
|
657
|
-
"%Y%m"
|
658
|
-
end
|
659
|
-
end
|
660
|
-
|
661
|
-
def round_date(date, period)
|
662
|
-
date = date.to_date
|
663
|
-
case period.to_sym
|
664
|
-
when :day
|
665
|
-
date
|
666
|
-
else
|
667
|
-
Date.new(date.year, date.month)
|
668
|
-
end
|
669
|
-
end
|
670
|
-
|
671
|
-
def advance_date(date, period, count = 1)
|
672
|
-
date = date.to_date
|
673
|
-
case period.to_sym
|
674
|
-
when :day
|
675
|
-
date.next_day(count)
|
676
|
-
else
|
677
|
-
date.next_month(count)
|
678
|
-
end
|
679
|
-
end
|
680
|
-
|
681
|
-
def quote_ident(value)
|
682
|
-
PG::Connection.quote_ident(value)
|
683
|
-
end
|
684
|
-
|
685
|
-
def quote_table(table)
|
686
|
-
table.split(".", 2).map { |v| quote_ident(v) }.join(".")
|
687
|
-
end
|
688
|
-
|
689
|
-
def quote_no_schema(table)
|
690
|
-
quote_ident(table.split(".", 2)[-1])
|
691
|
-
end
|
692
|
-
|
693
|
-
def regclass(table)
|
694
|
-
"'#{quote_table(table)}'::regclass"
|
695
|
-
end
|
696
|
-
|
697
|
-
def fetch_trigger(trigger_name, table)
|
698
|
-
execute("SELECT obj_description(oid, 'pg_trigger') AS comment FROM pg_trigger WHERE tgname = $1 AND tgrelid = #{regclass(table)}", [trigger_name])[0]
|
699
|
-
end
|
700
|
-
|
701
|
-
def qualify_table(table)
|
702
|
-
table.to_s.include?(".") ? table : [schema, table].join(".")
|
703
|
-
end
|
704
|
-
|
705
|
-
def settings_from_trigger(original_table, table)
|
706
|
-
trigger_name = self.trigger_name(original_table)
|
707
|
-
|
708
|
-
needs_comment = false
|
709
|
-
trigger_comment = fetch_trigger(trigger_name, table)
|
710
|
-
comment = trigger_comment || fetch_comment(table)
|
711
|
-
if comment
|
712
|
-
field, period, cast = comment["comment"].split(",").map { |v| v.split(":").last } rescue [nil, nil, nil]
|
713
|
-
end
|
714
|
-
|
715
|
-
unless period
|
716
|
-
needs_comment = true
|
717
|
-
function_def = execute("select pg_get_functiondef(oid) from pg_proc where proname = $1", [trigger_name])[0]
|
718
|
-
return [] unless function_def
|
719
|
-
function_def = function_def["pg_get_functiondef"]
|
720
|
-
sql_format = SQL_FORMAT.find { |_, f| function_def.include?("'#{f}'") }
|
721
|
-
return [] unless sql_format
|
722
|
-
period = sql_format[0]
|
723
|
-
field = /to_char\(NEW\.(\w+),/.match(function_def)[1]
|
724
|
-
end
|
725
|
-
|
726
|
-
# backwards compatibility with 0.2.3 and earlier (pre-timestamptz support)
|
727
|
-
unless cast
|
728
|
-
cast = "date"
|
729
|
-
# update comment to explicitly define cast
|
730
|
-
needs_comment = true
|
731
|
-
end
|
732
|
-
|
733
|
-
[period, field, cast, needs_comment, !trigger_comment]
|
734
|
-
end
|
735
|
-
|
736
|
-
def foreign_keys(table)
|
737
|
-
execute("SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid = #{regclass(table)} AND contype ='f'").map { |r| r["pg_get_constraintdef"] }
|
738
|
-
end
|
739
|
-
|
740
|
-
def server_version_num
|
741
|
-
execute("SHOW server_version_num").first["server_version_num"].to_i
|
742
|
-
end
|
743
|
-
end
|
744
|
-
end
|
6
|
+
# modules
|
7
|
+
require "pgslice/client"
|
8
|
+
require "pgslice/generic_table"
|
9
|
+
require "pgslice/table"
|
10
|
+
require "pgslice/version"
|