pgslice 0.4.2 → 0.4.7

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,32 @@
1
+ module PgSlice
2
+ class CLI < Thor
3
+ class << self
4
+ attr_accessor :instance
5
+ end
6
+
7
+ include Helpers
8
+
9
+ check_unknown_options!
10
+
11
+ class_option :url, desc: "Database URL"
12
+ class_option :dry_run, type: :boolean, default: false, desc: "Print statements without executing"
13
+
14
+ map %w[--version -v] => :version
15
+
16
+ def self.exit_on_failure?
17
+ ENV["PGSLICE_ENV"] != "test"
18
+ end
19
+
20
+ def initialize(*args)
21
+ PgSlice::CLI.instance = self
22
+ $stdout.sync = true
23
+ $stderr.sync = true
24
+ super
25
+ end
26
+
27
+ desc "version", "Show version"
28
+ def version
29
+ log("pgslice #{PgSlice::VERSION}")
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,133 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "add_partitions TABLE", "Add partitions"
4
+ option :intermediate, type: :boolean, default: false, desc: "Add to intermediate table"
5
+ option :past, type: :numeric, default: 0, desc: "Number of past partitions to add"
6
+ option :future, type: :numeric, default: 0, desc: "Number of future partitions to add"
7
+ option :tablespace, type: :string, default: "", desc: "Tablespace to use"
8
+ def add_partitions(table)
9
+ original_table = create_table(table)
10
+ table = options[:intermediate] ? original_table.intermediate_table : original_table
11
+ trigger_name = original_table.trigger_name
12
+
13
+ assert_table(table)
14
+
15
+ future = options[:future]
16
+ past = options[:past]
17
+ tablespace = options[:tablespace]
18
+ range = (-1 * past)..future
19
+
20
+ period, field, cast, needs_comment, declarative, version = table.fetch_settings(original_table.trigger_name)
21
+ unless period
22
+ message = "No settings found: #{table}"
23
+ message = "#{message}\nDid you mean to use --intermediate?" unless options[:intermediate]
24
+ abort message
25
+ end
26
+
27
+ queries = []
28
+
29
+ if needs_comment
30
+ queries << "COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_table(table)} is 'column:#{field},period:#{period},cast:#{cast}';"
31
+ end
32
+
33
+ # today = utc date
34
+ today = round_date(Time.now.utc.to_date, period)
35
+
36
+ schema_table =
37
+ if !declarative
38
+ table
39
+ elsif options[:intermediate]
40
+ original_table
41
+ else
42
+ table.partitions.last
43
+ end
44
+
45
+ # indexes automatically propagate in Postgres 11+
46
+ if version < 3
47
+ index_defs = schema_table.index_defs
48
+ fk_defs = schema_table.foreign_keys
49
+ else
50
+ index_defs = []
51
+ fk_defs = []
52
+ end
53
+
54
+ primary_key = schema_table.primary_key
55
+ tablespace_str = tablespace.empty? ? "" : " TABLESPACE #{quote_ident(tablespace)}"
56
+
57
+ added_partitions = []
58
+ range.each do |n|
59
+ day = advance_date(today, period, n)
60
+
61
+ partition = Table.new(original_table.schema, "#{original_table.name}_#{day.strftime(name_format(period))}")
62
+ next if partition.exists?
63
+ added_partitions << partition
64
+
65
+ if declarative
66
+ queries << <<-SQL
67
+ CREATE TABLE #{quote_table(partition)} PARTITION OF #{quote_table(table)} FOR VALUES FROM (#{sql_date(day, cast, false)}) TO (#{sql_date(advance_date(day, period, 1), cast, false)})#{tablespace_str};
68
+ SQL
69
+ else
70
+ queries << <<-SQL
71
+ CREATE TABLE #{quote_table(partition)}
72
+ (CHECK (#{quote_ident(field)} >= #{sql_date(day, cast)} AND #{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}))
73
+ INHERITS (#{quote_table(table)})#{tablespace_str};
74
+ SQL
75
+ end
76
+
77
+ queries << "ALTER TABLE #{quote_table(partition)} ADD PRIMARY KEY (#{primary_key.map { |k| quote_ident(k) }.join(", ")});" if primary_key.any?
78
+
79
+ index_defs.each do |index_def|
80
+ queries << make_index_def(index_def, partition)
81
+ end
82
+
83
+ fk_defs.each do |fk_def|
84
+ queries << make_fk_def(fk_def, partition)
85
+ end
86
+ end
87
+
88
+ unless declarative
89
+ # update trigger based on existing partitions
90
+ current_defs = []
91
+ future_defs = []
92
+ past_defs = []
93
+ name_format = self.name_format(period)
94
+ partitions = (table.partitions + added_partitions).uniq(&:name).sort_by(&:name)
95
+
96
+ partitions.each do |partition|
97
+ day = partition_date(partition, name_format)
98
+
99
+ sql = "(NEW.#{quote_ident(field)} >= #{sql_date(day, cast)} AND NEW.#{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}) THEN
100
+ INSERT INTO #{quote_table(partition)} VALUES (NEW.*);"
101
+
102
+ if day.to_date < today
103
+ past_defs << sql
104
+ elsif advance_date(day, period, 1) < today
105
+ current_defs << sql
106
+ else
107
+ future_defs << sql
108
+ end
109
+ end
110
+
111
+ # order by current period, future periods asc, past periods desc
112
+ trigger_defs = current_defs + future_defs + past_defs.reverse
113
+
114
+ if trigger_defs.any?
115
+ queries << <<-SQL
116
+ CREATE OR REPLACE FUNCTION #{quote_ident(trigger_name)}()
117
+ RETURNS trigger AS $$
118
+ BEGIN
119
+ IF #{trigger_defs.join("\n ELSIF ")}
120
+ ELSE
121
+ RAISE EXCEPTION 'Date out of range. Ensure partitions are created.';
122
+ END IF;
123
+ RETURN NULL;
124
+ END;
125
+ $$ LANGUAGE plpgsql;
126
+ SQL
127
+ end
128
+ end
129
+
130
+ run_queries(queries) if queries.any?
131
+ end
132
+ end
133
+ end
@@ -0,0 +1,13 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "analyze TABLE", "Analyze tables"
4
+ option :swapped, type: :boolean, default: false, desc: "Use swapped table"
5
+ def analyze(table)
6
+ table = create_table(table)
7
+ parent_table = options[:swapped] ? table : table.intermediate_table
8
+
9
+ analyze_list = parent_table.partitions + [parent_table]
10
+ run_queries_without_transaction(analyze_list.map { |t| "ANALYZE VERBOSE #{quote_table(t)};" })
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,103 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "fill TABLE", "Fill the partitions in batches"
4
+ option :batch_size, type: :numeric, default: 10000, desc: "Batch size"
5
+ option :swapped, type: :boolean, default: false, desc: "Use swapped table"
6
+ option :source_table, desc: "Source table"
7
+ option :dest_table, desc: "Destination table"
8
+ option :start, type: :numeric, desc: "Primary key to start"
9
+ option :where, desc: "Conditions to filter"
10
+ option :sleep, type: :numeric, desc: "Seconds to sleep between batches"
11
+ def fill(table)
12
+ table = create_table(table)
13
+ source_table = create_table(options[:source_table]) if options[:source_table]
14
+ dest_table = create_table(options[:dest_table]) if options[:dest_table]
15
+
16
+ if options[:swapped]
17
+ source_table ||= table.retired_table
18
+ dest_table ||= table
19
+ else
20
+ source_table ||= table
21
+ dest_table ||= table.intermediate_table
22
+ end
23
+
24
+ assert_table(source_table)
25
+ assert_table(dest_table)
26
+
27
+ period, field, cast, _, declarative, _ = dest_table.fetch_settings(table.trigger_name)
28
+
29
+ if period
30
+ name_format = self.name_format(period)
31
+
32
+ partitions = dest_table.partitions
33
+ if partitions.any?
34
+ starting_time = partition_date(partitions.first, name_format)
35
+ ending_time = advance_date(partition_date(partitions.last, name_format), period, 1)
36
+ end
37
+ end
38
+
39
+ schema_table = period && declarative ? partitions.last : table
40
+
41
+ primary_key = schema_table.primary_key[0]
42
+ abort "No primary key" unless primary_key
43
+
44
+ max_source_id = nil
45
+ begin
46
+ max_source_id = source_table.max_id(primary_key)
47
+ rescue PG::UndefinedFunction
48
+ abort "Only numeric primary keys are supported"
49
+ end
50
+
51
+ max_dest_id =
52
+ if options[:start]
53
+ options[:start]
54
+ elsif options[:swapped]
55
+ dest_table.max_id(primary_key, where: options[:where], below: max_source_id)
56
+ else
57
+ dest_table.max_id(primary_key, where: options[:where])
58
+ end
59
+
60
+ if max_dest_id == 0 && !options[:swapped]
61
+ min_source_id = source_table.min_id(primary_key, field, cast, starting_time, options[:where])
62
+ max_dest_id = min_source_id - 1 if min_source_id
63
+ end
64
+
65
+ starting_id = max_dest_id
66
+ fields = source_table.columns.map { |c| quote_ident(c) }.join(", ")
67
+ batch_size = options[:batch_size]
68
+
69
+ i = 1
70
+ batch_count = ((max_source_id - starting_id) / batch_size.to_f).ceil
71
+
72
+ if batch_count == 0
73
+ log_sql "/* nothing to fill */"
74
+ end
75
+
76
+ while starting_id < max_source_id
77
+ where = "#{quote_ident(primary_key)} > #{starting_id} AND #{quote_ident(primary_key)} <= #{starting_id + batch_size}"
78
+ if starting_time
79
+ where << " AND #{quote_ident(field)} >= #{sql_date(starting_time, cast)} AND #{quote_ident(field)} < #{sql_date(ending_time, cast)}"
80
+ end
81
+ if options[:where]
82
+ where << " AND #{options[:where]}"
83
+ end
84
+
85
+ query = <<-SQL
86
+ /* #{i} of #{batch_count} */
87
+ INSERT INTO #{quote_table(dest_table)} (#{fields})
88
+ SELECT #{fields} FROM #{quote_table(source_table)}
89
+ WHERE #{where}
90
+ SQL
91
+
92
+ run_query(query)
93
+
94
+ starting_id += batch_size
95
+ i += 1
96
+
97
+ if options[:sleep] && starting_id <= max_source_id
98
+ sleep(options[:sleep])
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
@@ -0,0 +1,97 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "prep TABLE [COLUMN] [PERIOD]", "Create an intermediate table for partitioning"
4
+ option :partition, type: :boolean, default: true, desc: "Partition the table"
5
+ option :trigger_based, type: :boolean, default: false, desc: "Use trigger-based partitioning"
6
+ def prep(table, column=nil, period=nil)
7
+ table = create_table(table)
8
+ intermediate_table = table.intermediate_table
9
+ trigger_name = table.trigger_name
10
+
11
+ unless options[:partition]
12
+ abort "Usage: \"pgslice prep TABLE --no-partition\"" if column || period
13
+ abort "Can't use --trigger-based and --no-partition" if options[:trigger_based]
14
+ end
15
+ assert_table(table)
16
+ assert_no_table(intermediate_table)
17
+
18
+ if options[:partition]
19
+ abort "Usage: \"pgslice prep TABLE COLUMN PERIOD\"" if !(column && period)
20
+ abort "Column not found: #{column}" unless table.columns.include?(column)
21
+ abort "Invalid period: #{period}" unless SQL_FORMAT[period.to_sym]
22
+ end
23
+
24
+ queries = []
25
+
26
+ # version summary
27
+ # 1. trigger-based
28
+ # 2. declarative, with indexes and foreign keys on child tables
29
+ # 3. declarative, with indexes and foreign keys on parent table
30
+ version =
31
+ if options[:trigger_based] || server_version_num < 100000
32
+ 1
33
+ elsif server_version_num < 110000
34
+ 2
35
+ else
36
+ 3
37
+ end
38
+
39
+ declarative = version > 1
40
+
41
+ if declarative && options[:partition]
42
+ queries << <<-SQL
43
+ CREATE TABLE #{quote_table(intermediate_table)} (LIKE #{quote_table(table)} INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING STORAGE INCLUDING COMMENTS) PARTITION BY RANGE (#{quote_ident(column)});
44
+ SQL
45
+
46
+ if version == 3
47
+ index_defs = table.index_defs
48
+ index_defs.each do |index_def|
49
+ queries << make_index_def(index_def, intermediate_table)
50
+ end
51
+
52
+ table.foreign_keys.each do |fk_def|
53
+ queries << make_fk_def(fk_def, intermediate_table)
54
+ end
55
+ end
56
+
57
+ # add comment
58
+ cast = table.column_cast(column)
59
+ queries << <<-SQL
60
+ COMMENT ON TABLE #{quote_table(intermediate_table)} is 'column:#{column},period:#{period},cast:#{cast},version:#{version}';
61
+ SQL
62
+ else
63
+ queries << <<-SQL
64
+ CREATE TABLE #{quote_table(intermediate_table)} (LIKE #{quote_table(table)} INCLUDING ALL);
65
+ SQL
66
+
67
+ table.foreign_keys.each do |fk_def|
68
+ queries << make_fk_def(fk_def, intermediate_table)
69
+ end
70
+ end
71
+
72
+ if options[:partition] && !declarative
73
+ queries << <<-SQL
74
+ CREATE FUNCTION #{quote_ident(trigger_name)}()
75
+ RETURNS trigger AS $$
76
+ BEGIN
77
+ RAISE EXCEPTION 'Create partitions first.';
78
+ END;
79
+ $$ LANGUAGE plpgsql;
80
+ SQL
81
+
82
+ queries << <<-SQL
83
+ CREATE TRIGGER #{quote_ident(trigger_name)}
84
+ BEFORE INSERT ON #{quote_table(intermediate_table)}
85
+ FOR EACH ROW EXECUTE PROCEDURE #{quote_ident(trigger_name)}();
86
+ SQL
87
+
88
+ cast = table.column_cast(column)
89
+ queries << <<-SQL
90
+ COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_table(intermediate_table)} is 'column:#{column},period:#{period},cast:#{cast}';
91
+ SQL
92
+ end
93
+
94
+ run_queries(queries)
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,28 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "swap TABLE", "Swap the intermediate table with the original table"
4
+ option :lock_timeout, default: "5s", desc: "Lock timeout"
5
+ def swap(table)
6
+ table = create_table(table)
7
+ intermediate_table = table.intermediate_table
8
+ retired_table = table.retired_table
9
+
10
+ assert_table(table)
11
+ assert_table(intermediate_table)
12
+ assert_no_table(retired_table)
13
+
14
+ queries = [
15
+ "ALTER TABLE #{quote_table(table)} RENAME TO #{quote_no_schema(retired_table)};",
16
+ "ALTER TABLE #{quote_table(intermediate_table)} RENAME TO #{quote_no_schema(table)};"
17
+ ]
18
+
19
+ table.sequences.each do |sequence|
20
+ queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_schema"])}.#{quote_ident(sequence["sequence_name"])} OWNED BY #{quote_table(table)}.#{quote_ident(sequence["related_column"])};"
21
+ end
22
+
23
+ queries.unshift("SET LOCAL lock_timeout = '#{options[:lock_timeout]}';") if server_version_num >= 90300
24
+
25
+ run_queries(queries)
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,18 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "unprep TABLE", "Undo prep"
4
+ def unprep(table)
5
+ table = create_table(table)
6
+ intermediate_table = table.intermediate_table
7
+ trigger_name = table.trigger_name
8
+
9
+ assert_table(intermediate_table)
10
+
11
+ queries = [
12
+ "DROP TABLE #{quote_table(intermediate_table)} CASCADE;",
13
+ "DROP FUNCTION IF EXISTS #{quote_ident(trigger_name)}();"
14
+ ]
15
+ run_queries(queries)
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,25 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "unswap TABLE", "Undo swap"
4
+ def unswap(table)
5
+ table = create_table(table)
6
+ intermediate_table = table.intermediate_table
7
+ retired_table = table.retired_table
8
+
9
+ assert_table(table)
10
+ assert_table(retired_table)
11
+ assert_no_table(intermediate_table)
12
+
13
+ queries = [
14
+ "ALTER TABLE #{quote_table(table)} RENAME TO #{quote_no_schema(intermediate_table)};",
15
+ "ALTER TABLE #{quote_table(retired_table)} RENAME TO #{quote_no_schema(table)};"
16
+ ]
17
+
18
+ table.sequences.each do |sequence|
19
+ queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_schema"])}.#{quote_ident(sequence["sequence_name"])} OWNED BY #{quote_table(table)}.#{quote_ident(sequence["related_column"])};"
20
+ end
21
+
22
+ run_queries(queries)
23
+ end
24
+ end
25
+ end