pgslice 0.4.2 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,32 @@
1
+ module PgSlice
2
+ class CLI < Thor
3
+ class << self
4
+ attr_accessor :instance
5
+ end
6
+
7
+ include Helpers
8
+
9
+ check_unknown_options!
10
+
11
+ class_option :url, desc: "Database URL"
12
+ class_option :dry_run, type: :boolean, default: false, desc: "Print statements without executing"
13
+
14
+ map %w[--version -v] => :version
15
+
16
+ def self.exit_on_failure?
17
+ ENV["PGSLICE_ENV"] != "test"
18
+ end
19
+
20
+ def initialize(*args)
21
+ PgSlice::CLI.instance = self
22
+ $stdout.sync = true
23
+ $stderr.sync = true
24
+ super
25
+ end
26
+
27
+ desc "version", "Show version"
28
+ def version
29
+ log("pgslice #{PgSlice::VERSION}")
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,133 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "add_partitions TABLE", "Add partitions"
4
+ option :intermediate, type: :boolean, default: false, desc: "Add to intermediate table"
5
+ option :past, type: :numeric, default: 0, desc: "Number of past partitions to add"
6
+ option :future, type: :numeric, default: 0, desc: "Number of future partitions to add"
7
+ option :tablespace, type: :string, default: "", desc: "Tablespace to use"
8
+ def add_partitions(table)
9
+ original_table = create_table(table)
10
+ table = options[:intermediate] ? original_table.intermediate_table : original_table
11
+ trigger_name = original_table.trigger_name
12
+
13
+ assert_table(table)
14
+
15
+ future = options[:future]
16
+ past = options[:past]
17
+ tablespace = options[:tablespace]
18
+ range = (-1 * past)..future
19
+
20
+ period, field, cast, needs_comment, declarative, version = table.fetch_settings(original_table.trigger_name)
21
+ unless period
22
+ message = "No settings found: #{table}"
23
+ message = "#{message}\nDid you mean to use --intermediate?" unless options[:intermediate]
24
+ abort message
25
+ end
26
+
27
+ queries = []
28
+
29
+ if needs_comment
30
+ queries << "COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_table(table)} is 'column:#{field},period:#{period},cast:#{cast}';"
31
+ end
32
+
33
+ # today = utc date
34
+ today = round_date(Time.now.utc.to_date, period)
35
+
36
+ schema_table =
37
+ if !declarative
38
+ table
39
+ elsif options[:intermediate]
40
+ original_table
41
+ else
42
+ table.partitions.last
43
+ end
44
+
45
+ # indexes automatically propagate in Postgres 11+
46
+ if version < 3
47
+ index_defs = schema_table.index_defs
48
+ fk_defs = schema_table.foreign_keys
49
+ else
50
+ index_defs = []
51
+ fk_defs = []
52
+ end
53
+
54
+ primary_key = schema_table.primary_key
55
+ tablespace_str = tablespace.empty? ? "" : " TABLESPACE #{quote_ident(tablespace)}"
56
+
57
+ added_partitions = []
58
+ range.each do |n|
59
+ day = advance_date(today, period, n)
60
+
61
+ partition = Table.new(original_table.schema, "#{original_table.name}_#{day.strftime(name_format(period))}")
62
+ next if partition.exists?
63
+ added_partitions << partition
64
+
65
+ if declarative
66
+ queries << <<-SQL
67
+ CREATE TABLE #{quote_table(partition)} PARTITION OF #{quote_table(table)} FOR VALUES FROM (#{sql_date(day, cast, false)}) TO (#{sql_date(advance_date(day, period, 1), cast, false)})#{tablespace_str};
68
+ SQL
69
+ else
70
+ queries << <<-SQL
71
+ CREATE TABLE #{quote_table(partition)}
72
+ (CHECK (#{quote_ident(field)} >= #{sql_date(day, cast)} AND #{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}))
73
+ INHERITS (#{quote_table(table)})#{tablespace_str};
74
+ SQL
75
+ end
76
+
77
+ queries << "ALTER TABLE #{quote_table(partition)} ADD PRIMARY KEY (#{primary_key.map { |k| quote_ident(k) }.join(", ")});" if primary_key.any?
78
+
79
+ index_defs.each do |index_def|
80
+ queries << make_index_def(index_def, partition)
81
+ end
82
+
83
+ fk_defs.each do |fk_def|
84
+ queries << make_fk_def(fk_def, partition)
85
+ end
86
+ end
87
+
88
+ unless declarative
89
+ # update trigger based on existing partitions
90
+ current_defs = []
91
+ future_defs = []
92
+ past_defs = []
93
+ name_format = self.name_format(period)
94
+ partitions = (table.partitions + added_partitions).uniq(&:name).sort_by(&:name)
95
+
96
+ partitions.each do |partition|
97
+ day = partition_date(partition, name_format)
98
+
99
+ sql = "(NEW.#{quote_ident(field)} >= #{sql_date(day, cast)} AND NEW.#{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}) THEN
100
+ INSERT INTO #{quote_table(partition)} VALUES (NEW.*);"
101
+
102
+ if day.to_date < today
103
+ past_defs << sql
104
+ elsif advance_date(day, period, 1) < today
105
+ current_defs << sql
106
+ else
107
+ future_defs << sql
108
+ end
109
+ end
110
+
111
+ # order by current period, future periods asc, past periods desc
112
+ trigger_defs = current_defs + future_defs + past_defs.reverse
113
+
114
+ if trigger_defs.any?
115
+ queries << <<-SQL
116
+ CREATE OR REPLACE FUNCTION #{quote_ident(trigger_name)}()
117
+ RETURNS trigger AS $$
118
+ BEGIN
119
+ IF #{trigger_defs.join("\n ELSIF ")}
120
+ ELSE
121
+ RAISE EXCEPTION 'Date out of range. Ensure partitions are created.';
122
+ END IF;
123
+ RETURN NULL;
124
+ END;
125
+ $$ LANGUAGE plpgsql;
126
+ SQL
127
+ end
128
+ end
129
+
130
+ run_queries(queries) if queries.any?
131
+ end
132
+ end
133
+ end
@@ -0,0 +1,13 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "analyze TABLE", "Analyze tables"
4
+ option :swapped, type: :boolean, default: false, desc: "Use swapped table"
5
+ def analyze(table)
6
+ table = create_table(table)
7
+ parent_table = options[:swapped] ? table : table.intermediate_table
8
+
9
+ analyze_list = parent_table.partitions + [parent_table]
10
+ run_queries_without_transaction(analyze_list.map { |t| "ANALYZE VERBOSE #{quote_table(t)};" })
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,103 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "fill TABLE", "Fill the partitions in batches"
4
+ option :batch_size, type: :numeric, default: 10000, desc: "Batch size"
5
+ option :swapped, type: :boolean, default: false, desc: "Use swapped table"
6
+ option :source_table, desc: "Source table"
7
+ option :dest_table, desc: "Destination table"
8
+ option :start, type: :numeric, desc: "Primary key to start"
9
+ option :where, desc: "Conditions to filter"
10
+ option :sleep, type: :numeric, desc: "Seconds to sleep between batches"
11
+ def fill(table)
12
+ table = create_table(table)
13
+ source_table = create_table(options[:source_table]) if options[:source_table]
14
+ dest_table = create_table(options[:dest_table]) if options[:dest_table]
15
+
16
+ if options[:swapped]
17
+ source_table ||= table.retired_table
18
+ dest_table ||= table
19
+ else
20
+ source_table ||= table
21
+ dest_table ||= table.intermediate_table
22
+ end
23
+
24
+ assert_table(source_table)
25
+ assert_table(dest_table)
26
+
27
+ period, field, cast, _, declarative, _ = dest_table.fetch_settings(table.trigger_name)
28
+
29
+ if period
30
+ name_format = self.name_format(period)
31
+
32
+ partitions = dest_table.partitions
33
+ if partitions.any?
34
+ starting_time = partition_date(partitions.first, name_format)
35
+ ending_time = advance_date(partition_date(partitions.last, name_format), period, 1)
36
+ end
37
+ end
38
+
39
+ schema_table = period && declarative ? partitions.last : table
40
+
41
+ primary_key = schema_table.primary_key[0]
42
+ abort "No primary key" unless primary_key
43
+
44
+ max_source_id = nil
45
+ begin
46
+ max_source_id = source_table.max_id(primary_key)
47
+ rescue PG::UndefinedFunction
48
+ abort "Only numeric primary keys are supported"
49
+ end
50
+
51
+ max_dest_id =
52
+ if options[:start]
53
+ options[:start]
54
+ elsif options[:swapped]
55
+ dest_table.max_id(primary_key, where: options[:where], below: max_source_id)
56
+ else
57
+ dest_table.max_id(primary_key, where: options[:where])
58
+ end
59
+
60
+ if max_dest_id == 0 && !options[:swapped]
61
+ min_source_id = source_table.min_id(primary_key, field, cast, starting_time, options[:where])
62
+ max_dest_id = min_source_id - 1 if min_source_id
63
+ end
64
+
65
+ starting_id = max_dest_id
66
+ fields = source_table.columns.map { |c| quote_ident(c) }.join(", ")
67
+ batch_size = options[:batch_size]
68
+
69
+ i = 1
70
+ batch_count = ((max_source_id - starting_id) / batch_size.to_f).ceil
71
+
72
+ if batch_count == 0
73
+ log_sql "/* nothing to fill */"
74
+ end
75
+
76
+ while starting_id < max_source_id
77
+ where = "#{quote_ident(primary_key)} > #{starting_id} AND #{quote_ident(primary_key)} <= #{starting_id + batch_size}"
78
+ if starting_time
79
+ where << " AND #{quote_ident(field)} >= #{sql_date(starting_time, cast)} AND #{quote_ident(field)} < #{sql_date(ending_time, cast)}"
80
+ end
81
+ if options[:where]
82
+ where << " AND #{options[:where]}"
83
+ end
84
+
85
+ query = <<-SQL
86
+ /* #{i} of #{batch_count} */
87
+ INSERT INTO #{quote_table(dest_table)} (#{fields})
88
+ SELECT #{fields} FROM #{quote_table(source_table)}
89
+ WHERE #{where}
90
+ SQL
91
+
92
+ run_query(query)
93
+
94
+ starting_id += batch_size
95
+ i += 1
96
+
97
+ if options[:sleep] && starting_id <= max_source_id
98
+ sleep(options[:sleep])
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
@@ -0,0 +1,97 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "prep TABLE [COLUMN] [PERIOD]", "Create an intermediate table for partitioning"
4
+ option :partition, type: :boolean, default: true, desc: "Partition the table"
5
+ option :trigger_based, type: :boolean, default: false, desc: "Use trigger-based partitioning"
6
+ def prep(table, column=nil, period=nil)
7
+ table = create_table(table)
8
+ intermediate_table = table.intermediate_table
9
+ trigger_name = table.trigger_name
10
+
11
+ unless options[:partition]
12
+ abort "Usage: \"pgslice prep TABLE --no-partition\"" if column || period
13
+ abort "Can't use --trigger-based and --no-partition" if options[:trigger_based]
14
+ end
15
+ assert_table(table)
16
+ assert_no_table(intermediate_table)
17
+
18
+ if options[:partition]
19
+ abort "Usage: \"pgslice prep TABLE COLUMN PERIOD\"" if !(column && period)
20
+ abort "Column not found: #{column}" unless table.columns.include?(column)
21
+ abort "Invalid period: #{period}" unless SQL_FORMAT[period.to_sym]
22
+ end
23
+
24
+ queries = []
25
+
26
+ # version summary
27
+ # 1. trigger-based
28
+ # 2. declarative, with indexes and foreign keys on child tables
29
+ # 3. declarative, with indexes and foreign keys on parent table
30
+ version =
31
+ if options[:trigger_based] || server_version_num < 100000
32
+ 1
33
+ elsif server_version_num < 110000
34
+ 2
35
+ else
36
+ 3
37
+ end
38
+
39
+ declarative = version > 1
40
+
41
+ if declarative && options[:partition]
42
+ queries << <<-SQL
43
+ CREATE TABLE #{quote_table(intermediate_table)} (LIKE #{quote_table(table)} INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING STORAGE INCLUDING COMMENTS) PARTITION BY RANGE (#{quote_ident(column)});
44
+ SQL
45
+
46
+ if version == 3
47
+ index_defs = table.index_defs
48
+ index_defs.each do |index_def|
49
+ queries << make_index_def(index_def, intermediate_table)
50
+ end
51
+
52
+ table.foreign_keys.each do |fk_def|
53
+ queries << make_fk_def(fk_def, intermediate_table)
54
+ end
55
+ end
56
+
57
+ # add comment
58
+ cast = table.column_cast(column)
59
+ queries << <<-SQL
60
+ COMMENT ON TABLE #{quote_table(intermediate_table)} is 'column:#{column},period:#{period},cast:#{cast},version:#{version}';
61
+ SQL
62
+ else
63
+ queries << <<-SQL
64
+ CREATE TABLE #{quote_table(intermediate_table)} (LIKE #{quote_table(table)} INCLUDING ALL);
65
+ SQL
66
+
67
+ table.foreign_keys.each do |fk_def|
68
+ queries << make_fk_def(fk_def, intermediate_table)
69
+ end
70
+ end
71
+
72
+ if options[:partition] && !declarative
73
+ queries << <<-SQL
74
+ CREATE FUNCTION #{quote_ident(trigger_name)}()
75
+ RETURNS trigger AS $$
76
+ BEGIN
77
+ RAISE EXCEPTION 'Create partitions first.';
78
+ END;
79
+ $$ LANGUAGE plpgsql;
80
+ SQL
81
+
82
+ queries << <<-SQL
83
+ CREATE TRIGGER #{quote_ident(trigger_name)}
84
+ BEFORE INSERT ON #{quote_table(intermediate_table)}
85
+ FOR EACH ROW EXECUTE PROCEDURE #{quote_ident(trigger_name)}();
86
+ SQL
87
+
88
+ cast = table.column_cast(column)
89
+ queries << <<-SQL
90
+ COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_table(intermediate_table)} is 'column:#{column},period:#{period},cast:#{cast}';
91
+ SQL
92
+ end
93
+
94
+ run_queries(queries)
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,28 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "swap TABLE", "Swap the intermediate table with the original table"
4
+ option :lock_timeout, default: "5s", desc: "Lock timeout"
5
+ def swap(table)
6
+ table = create_table(table)
7
+ intermediate_table = table.intermediate_table
8
+ retired_table = table.retired_table
9
+
10
+ assert_table(table)
11
+ assert_table(intermediate_table)
12
+ assert_no_table(retired_table)
13
+
14
+ queries = [
15
+ "ALTER TABLE #{quote_table(table)} RENAME TO #{quote_no_schema(retired_table)};",
16
+ "ALTER TABLE #{quote_table(intermediate_table)} RENAME TO #{quote_no_schema(table)};"
17
+ ]
18
+
19
+ table.sequences.each do |sequence|
20
+ queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_schema"])}.#{quote_ident(sequence["sequence_name"])} OWNED BY #{quote_table(table)}.#{quote_ident(sequence["related_column"])};"
21
+ end
22
+
23
+ queries.unshift("SET LOCAL lock_timeout = '#{options[:lock_timeout]}';") if server_version_num >= 90300
24
+
25
+ run_queries(queries)
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,18 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "unprep TABLE", "Undo prep"
4
+ def unprep(table)
5
+ table = create_table(table)
6
+ intermediate_table = table.intermediate_table
7
+ trigger_name = table.trigger_name
8
+
9
+ assert_table(intermediate_table)
10
+
11
+ queries = [
12
+ "DROP TABLE #{quote_table(intermediate_table)} CASCADE;",
13
+ "DROP FUNCTION IF EXISTS #{quote_ident(trigger_name)}();"
14
+ ]
15
+ run_queries(queries)
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,25 @@
1
+ module PgSlice
2
+ class CLI
3
+ desc "unswap TABLE", "Undo swap"
4
+ def unswap(table)
5
+ table = create_table(table)
6
+ intermediate_table = table.intermediate_table
7
+ retired_table = table.retired_table
8
+
9
+ assert_table(table)
10
+ assert_table(retired_table)
11
+ assert_no_table(intermediate_table)
12
+
13
+ queries = [
14
+ "ALTER TABLE #{quote_table(table)} RENAME TO #{quote_no_schema(intermediate_table)};",
15
+ "ALTER TABLE #{quote_table(retired_table)} RENAME TO #{quote_no_schema(table)};"
16
+ ]
17
+
18
+ table.sequences.each do |sequence|
19
+ queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_schema"])}.#{quote_ident(sequence["sequence_name"])} OWNED BY #{quote_table(table)}.#{quote_ident(sequence["related_column"])};"
20
+ end
21
+
22
+ run_queries(queries)
23
+ end
24
+ end
25
+ end