pgslice 0.3.4 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +14 -0
- data/lib/pgslice.rb +43 -35
- data/lib/pgslice/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 43cc2dd89adcb7a9ab6967e5d87b28d24e7e8c26
|
4
|
+
data.tar.gz: f9661b8e3cdf0239c0ba2174bc0c5694f95d834a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 41176b19e45fd47d89d50f45b4a4cf9ab5802ed63b68100852e776dc0c821745a85cb80b2f942e5a445fa5752b696c0dce5a6fe6528477f26ec3673d41cbe94f
|
7
|
+
data.tar.gz: 04d50a9863351b4b97928e8dfb5f8a018c5723e5daa473a4430c5ed41ff9db355ebb4d1f749a48ebc5eb3ebc7e88e84a8ab795edd394fbd351219a09d30071b8
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -181,6 +181,20 @@ INSERT INTO visits_intermediate ("id", "user_id", "ip", "created_at")
|
|
181
181
|
WHERE id > 20000 AND id <= 30000 AND created_at >= '2016-08-01'::date AND created_at < '2016-11-01'::date
|
182
182
|
```
|
183
183
|
|
184
|
+
```sh
|
185
|
+
pgslice analyze visits
|
186
|
+
```
|
187
|
+
|
188
|
+
```sql
|
189
|
+
ANALYZE VERBOSE visits_201608;
|
190
|
+
|
191
|
+
ANALYZE VERBOSE visits_201609;
|
192
|
+
|
193
|
+
ANALYZE VERBOSE visits_201610;
|
194
|
+
|
195
|
+
ANALYZE VERBOSE visits_intermediate;
|
196
|
+
```
|
197
|
+
|
184
198
|
```sh
|
185
199
|
pgslice swap visits
|
186
200
|
```
|
data/lib/pgslice.rb
CHANGED
@@ -73,13 +73,13 @@ module PgSlice
|
|
73
73
|
queries = []
|
74
74
|
|
75
75
|
queries << <<-SQL
|
76
|
-
CREATE TABLE #{intermediate_table} (LIKE #{table} INCLUDING ALL);
|
76
|
+
CREATE TABLE #{quote_ident(intermediate_table)} (LIKE #{quote_ident(table)} INCLUDING ALL);
|
77
77
|
SQL
|
78
78
|
|
79
79
|
unless options[:no_partition]
|
80
80
|
sql_format = SQL_FORMAT[period.to_sym]
|
81
81
|
queries << <<-SQL
|
82
|
-
CREATE FUNCTION #{trigger_name}()
|
82
|
+
CREATE FUNCTION #{quote_ident(trigger_name)}()
|
83
83
|
RETURNS trigger AS $$
|
84
84
|
BEGIN
|
85
85
|
RAISE EXCEPTION 'Create partitions first.';
|
@@ -88,14 +88,14 @@ CREATE FUNCTION #{trigger_name}()
|
|
88
88
|
SQL
|
89
89
|
|
90
90
|
queries << <<-SQL
|
91
|
-
CREATE TRIGGER #{trigger_name}
|
92
|
-
BEFORE INSERT ON #{intermediate_table}
|
93
|
-
FOR EACH ROW EXECUTE PROCEDURE #{trigger_name}();
|
91
|
+
CREATE TRIGGER #{quote_ident(trigger_name)}
|
92
|
+
BEFORE INSERT ON #{quote_ident(intermediate_table)}
|
93
|
+
FOR EACH ROW EXECUTE PROCEDURE #{quote_ident(trigger_name)}();
|
94
94
|
SQL
|
95
95
|
|
96
96
|
cast = column_cast(table, column)
|
97
97
|
queries << <<-SQL
|
98
|
-
COMMENT ON TRIGGER #{trigger_name} ON #{intermediate_table} is 'column:#{column},period:#{period},cast:#{cast}';
|
98
|
+
COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_ident(intermediate_table)} is 'column:#{column},period:#{period},cast:#{cast}';
|
99
99
|
SQL
|
100
100
|
end
|
101
101
|
|
@@ -111,7 +111,7 @@ SQL
|
|
111
111
|
abort "Table not found: #{intermediate_table}" unless table_exists?(intermediate_table)
|
112
112
|
|
113
113
|
queries = [
|
114
|
-
"DROP TABLE #{intermediate_table} CASCADE;",
|
114
|
+
"DROP TABLE #{quote_ident(intermediate_table)} CASCADE;",
|
115
115
|
"DROP FUNCTION IF EXISTS #{trigger_name}();"
|
116
116
|
]
|
117
117
|
run_queries(queries)
|
@@ -132,7 +132,7 @@ SQL
|
|
132
132
|
# ensure table has trigger
|
133
133
|
abort "No trigger on table: #{table}\nDid you mean to use --intermediate?" unless has_trigger?(trigger_name, table)
|
134
134
|
|
135
|
-
index_defs = execute("
|
135
|
+
index_defs = execute("SELECT pg_get_indexdef(indexrelid) FROM pg_index INNER JOIN pg_stat_user_indexes USING (indexrelid) WHERE relname = $1 AND schemaname = $2 AND indisprimary = 'f'", [original_table, schema]).map { |r| r["pg_get_indexdef"] }
|
136
136
|
primary_key = self.primary_key(table)
|
137
137
|
|
138
138
|
queries = []
|
@@ -141,7 +141,7 @@ SQL
|
|
141
141
|
abort "Could not read settings" unless period
|
142
142
|
|
143
143
|
if needs_comment
|
144
|
-
queries << "COMMENT ON TRIGGER #{trigger_name} ON #{table} is 'column:#{field},period:#{period},cast:#{cast}';"
|
144
|
+
queries << "COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_ident(table)} is 'column:#{field},period:#{period},cast:#{cast}';"
|
145
145
|
end
|
146
146
|
|
147
147
|
# today = utc date
|
@@ -155,12 +155,12 @@ SQL
|
|
155
155
|
added_partitions << partition_name
|
156
156
|
|
157
157
|
queries << <<-SQL
|
158
|
-
CREATE TABLE #{partition_name}
|
159
|
-
(CHECK (#{field} >= #{sql_date(day, cast)} AND #{field} < #{sql_date(advance_date(day, period, 1), cast)}))
|
160
|
-
INHERITS (#{table});
|
158
|
+
CREATE TABLE #{quote_ident(partition_name)}
|
159
|
+
(CHECK (#{quote_ident(field)} >= #{sql_date(day, cast)} AND #{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}))
|
160
|
+
INHERITS (#{quote_ident(table)});
|
161
161
|
SQL
|
162
162
|
|
163
|
-
queries << "ALTER TABLE #{partition_name} ADD PRIMARY KEY (#{primary_key});" if primary_key
|
163
|
+
queries << "ALTER TABLE #{quote_ident(partition_name)} ADD PRIMARY KEY (#{quote_ident(primary_key)});" if primary_key
|
164
164
|
|
165
165
|
index_defs.each do |index_def|
|
166
166
|
queries << index_def.sub(" ON #{original_table} USING ", " ON #{partition_name} USING ").sub(/ INDEX .+ ON /, " INDEX ON ") + ";"
|
@@ -179,8 +179,8 @@ CREATE TABLE #{partition_name}
|
|
179
179
|
day = DateTime.strptime(table.split("_").last, name_format)
|
180
180
|
partition_name = "#{original_table}_#{day.strftime(name_format(period))}"
|
181
181
|
|
182
|
-
sql = "(NEW.#{field} >= #{sql_date(day, cast)} AND NEW.#{field} < #{sql_date(advance_date(day, period, 1), cast)}) THEN
|
183
|
-
INSERT INTO #{partition_name} VALUES (NEW.*);"
|
182
|
+
sql = "(NEW.#{quote_ident(field)} >= #{sql_date(day, cast)} AND NEW.#{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}) THEN
|
183
|
+
INSERT INTO #{quote_ident(partition_name)} VALUES (NEW.*);"
|
184
184
|
|
185
185
|
if day.to_date < today
|
186
186
|
past_defs << sql
|
@@ -196,7 +196,7 @@ CREATE TABLE #{partition_name}
|
|
196
196
|
|
197
197
|
if trigger_defs.any?
|
198
198
|
queries << <<-SQL
|
199
|
-
CREATE OR REPLACE FUNCTION #{trigger_name}()
|
199
|
+
CREATE OR REPLACE FUNCTION #{quote_ident(trigger_name)}()
|
200
200
|
RETURNS trigger AS $$
|
201
201
|
BEGIN
|
202
202
|
IF #{trigger_defs.join("\n ELSIF ")}
|
@@ -262,7 +262,7 @@ CREATE OR REPLACE FUNCTION #{trigger_name}()
|
|
262
262
|
end
|
263
263
|
|
264
264
|
starting_id = max_dest_id
|
265
|
-
fields = columns(source_table).map { |c|
|
265
|
+
fields = columns(source_table).map { |c| quote_ident(c) }.join(", ")
|
266
266
|
batch_size = options[:batch_size]
|
267
267
|
|
268
268
|
i = 1
|
@@ -273,9 +273,9 @@ CREATE OR REPLACE FUNCTION #{trigger_name}()
|
|
273
273
|
end
|
274
274
|
|
275
275
|
while starting_id < max_source_id
|
276
|
-
where = "#{primary_key} > #{starting_id} AND #{primary_key} <= #{starting_id + batch_size}"
|
276
|
+
where = "#{quote_ident(primary_key)} > #{starting_id} AND #{quote_ident(primary_key)} <= #{starting_id + batch_size}"
|
277
277
|
if starting_time
|
278
|
-
where << " AND #{field} >= #{sql_date(starting_time, cast)} AND #{field} < #{sql_date(ending_time, cast)}"
|
278
|
+
where << " AND #{quote_ident(field)} >= #{sql_date(starting_time, cast)} AND #{quote_ident(field)} < #{sql_date(ending_time, cast)}"
|
279
279
|
end
|
280
280
|
if options[:where]
|
281
281
|
where << " AND #{options[:where]}"
|
@@ -283,8 +283,8 @@ CREATE OR REPLACE FUNCTION #{trigger_name}()
|
|
283
283
|
|
284
284
|
query = <<-SQL
|
285
285
|
/* #{i} of #{batch_count} */
|
286
|
-
INSERT INTO #{dest_table} (#{fields})
|
287
|
-
SELECT #{fields} FROM #{source_table}
|
286
|
+
INSERT INTO #{quote_ident(dest_table)} (#{fields})
|
287
|
+
SELECT #{fields} FROM #{quote_ident(source_table)}
|
288
288
|
WHERE #{where}
|
289
289
|
SQL
|
290
290
|
|
@@ -310,12 +310,12 @@ INSERT INTO #{dest_table} (#{fields})
|
|
310
310
|
abort "Table already exists: #{retired_table}" if table_exists?(retired_table)
|
311
311
|
|
312
312
|
queries = [
|
313
|
-
"ALTER TABLE #{table} RENAME TO #{retired_table};",
|
314
|
-
"ALTER TABLE #{intermediate_table} RENAME TO #{table};"
|
313
|
+
"ALTER TABLE #{quote_ident(table)} RENAME TO #{quote_ident(retired_table)};",
|
314
|
+
"ALTER TABLE #{quote_ident(intermediate_table)} RENAME TO #{quote_ident(table)};"
|
315
315
|
]
|
316
316
|
|
317
317
|
self.sequences(table).each do |sequence|
|
318
|
-
queries << "ALTER SEQUENCE #{sequence["sequence_name"]} OWNED BY #{table}.#{sequence["related_column"]};"
|
318
|
+
queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_name"])} OWNED BY #{table}.#{sequence["related_column"]};"
|
319
319
|
end
|
320
320
|
|
321
321
|
queries.unshift("SET LOCAL lock_timeout = '#{options[:lock_timeout]}';") if server_version_num >= 90300
|
@@ -334,12 +334,12 @@ INSERT INTO #{dest_table} (#{fields})
|
|
334
334
|
abort "Table already exists: #{intermediate_table}" if table_exists?(intermediate_table)
|
335
335
|
|
336
336
|
queries = [
|
337
|
-
"ALTER TABLE #{table} RENAME TO #{intermediate_table};",
|
338
|
-
"ALTER TABLE #{retired_table} RENAME TO #{table};"
|
337
|
+
"ALTER TABLE #{quote_ident(table)} RENAME TO #{quote_ident(intermediate_table)};",
|
338
|
+
"ALTER TABLE #{quote_ident(retired_table)} RENAME TO #{quote_ident(table)};"
|
339
339
|
]
|
340
340
|
|
341
341
|
self.sequences(table).each do |sequence|
|
342
|
-
queries << "ALTER SEQUENCE #{sequence["sequence_name"]} OWNED BY #{table}.#{sequence["related_column"]};"
|
342
|
+
queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_name"])} OWNED BY #{table}.#{sequence["related_column"]};"
|
343
343
|
end
|
344
344
|
|
345
345
|
run_queries(queries)
|
@@ -353,7 +353,7 @@ INSERT INTO #{dest_table} (#{fields})
|
|
353
353
|
|
354
354
|
existing_tables = self.existing_tables(like: "#{table}_%").select { |t| /\A#{Regexp.escape("#{table}_")}\d{6,8}\z/.match(t) }
|
355
355
|
analyze_list = existing_tables + [parent_table]
|
356
|
-
run_queries_without_transaction analyze_list.map { |t| "ANALYZE VERBOSE #{t};" }
|
356
|
+
run_queries_without_transaction analyze_list.map { |t| "ANALYZE VERBOSE #{quote_ident(t)};" }
|
357
357
|
end
|
358
358
|
|
359
359
|
# arguments
|
@@ -486,7 +486,7 @@ INSERT INTO #{dest_table} (#{fields})
|
|
486
486
|
FROM
|
487
487
|
pg_index, pg_class, pg_attribute, pg_namespace
|
488
488
|
WHERE
|
489
|
-
|
489
|
+
relname = $2 AND
|
490
490
|
indrelid = pg_class.oid AND
|
491
491
|
nspname = $1 AND
|
492
492
|
pg_class.relnamespace = pg_namespace.oid AND
|
@@ -499,25 +499,25 @@ INSERT INTO #{dest_table} (#{fields})
|
|
499
499
|
end
|
500
500
|
|
501
501
|
def max_id(table, primary_key, below: nil, where: nil)
|
502
|
-
query = "SELECT MAX(#{primary_key}) FROM #{table}"
|
502
|
+
query = "SELECT MAX(#{quote_ident(primary_key)}) FROM #{quote_ident(table)}"
|
503
503
|
conditions = []
|
504
|
-
conditions << "#{primary_key} <= #{below}" if below
|
504
|
+
conditions << "#{quote_ident(primary_key)} <= #{below}" if below
|
505
505
|
conditions << where if where
|
506
506
|
query << " WHERE #{conditions.join(" AND ")}" if conditions.any?
|
507
507
|
execute(query)[0]["max"].to_i
|
508
508
|
end
|
509
509
|
|
510
510
|
def min_id(table, primary_key, column, cast, starting_time, where)
|
511
|
-
query = "SELECT MIN(#{primary_key}) FROM #{table}"
|
511
|
+
query = "SELECT MIN(#{quote_ident(primary_key)}) FROM #{quote_ident(table)}"
|
512
512
|
conditions = []
|
513
|
-
conditions << "#{column} >= #{sql_date(starting_time, cast)}" if starting_time
|
513
|
+
conditions << "#{quote_ident(column)} >= #{sql_date(starting_time, cast)}" if starting_time
|
514
514
|
conditions << where if where
|
515
515
|
query << " WHERE #{conditions.join(" AND ")}" if conditions.any?
|
516
516
|
(execute(query)[0]["min"] || 1).to_i
|
517
517
|
end
|
518
518
|
|
519
519
|
def has_trigger?(trigger_name, table)
|
520
|
-
|
520
|
+
!fetch_trigger(trigger_name, table).nil?
|
521
521
|
end
|
522
522
|
|
523
523
|
# http://www.dbforums.com/showthread.php?1667561-How-to-list-sequences-and-the-columns-by-SQL
|
@@ -595,11 +595,19 @@ INSERT INTO #{dest_table} (#{fields})
|
|
595
595
|
end
|
596
596
|
end
|
597
597
|
|
598
|
+
def quote_ident(value)
|
599
|
+
PG::Connection.quote_ident(value)
|
600
|
+
end
|
601
|
+
|
602
|
+
def fetch_trigger(trigger_name, table)
|
603
|
+
execute("SELECT obj_description(oid, 'pg_trigger') AS comment FROM pg_trigger WHERE tgname = $1 AND EXISTS (SELECT 1 FROM pg_stat_user_tables WHERE relid = tgrelid AND relname = $2 AND schemaname = $3)", [trigger_name, table, schema])[0]
|
604
|
+
end
|
605
|
+
|
598
606
|
def settings_from_trigger(original_table, table)
|
599
607
|
trigger_name = self.trigger_name(original_table)
|
600
608
|
|
601
609
|
needs_comment = false
|
602
|
-
comment =
|
610
|
+
comment = fetch_trigger(trigger_name, table)
|
603
611
|
if comment
|
604
612
|
field, period, cast = comment["comment"].split(",").map { |v| v.split(":").last } rescue [nil, nil, nil]
|
605
613
|
end
|
data/lib/pgslice/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pgslice
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Kane
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-07-
|
11
|
+
date: 2017-07-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: slop
|