pgslice 0.4.1 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +47 -21
- data/LICENSE.txt +1 -1
- data/README.md +107 -131
- data/exe/pgslice +4 -7
- data/lib/pgslice.rb +18 -723
- data/lib/pgslice/cli.rb +32 -0
- data/lib/pgslice/cli/add_partitions.rb +130 -0
- data/lib/pgslice/cli/analyze.rb +13 -0
- data/lib/pgslice/cli/fill.rb +103 -0
- data/lib/pgslice/cli/prep.rb +97 -0
- data/lib/pgslice/cli/swap.rb +28 -0
- data/lib/pgslice/cli/unprep.rb +18 -0
- data/lib/pgslice/cli/unswap.rb +25 -0
- data/lib/pgslice/helpers.rb +176 -0
- data/lib/pgslice/table.rb +196 -0
- data/lib/pgslice/version.rb +1 -1
- metadata +20 -18
- data/.gitignore +0 -9
- data/.travis.yml +0 -13
- data/Dockerfile +0 -3
- data/Gemfile +0 -4
- data/Rakefile +0 -11
- data/pgslice.gemspec +0 -27
@@ -0,0 +1,176 @@
|
|
1
|
+
module PgSlice
|
2
|
+
module Helpers
|
3
|
+
SQL_FORMAT = {
|
4
|
+
day: "YYYYMMDD",
|
5
|
+
month: "YYYYMM",
|
6
|
+
year: "YYYY"
|
7
|
+
}
|
8
|
+
|
9
|
+
protected
|
10
|
+
|
11
|
+
# output
|
12
|
+
|
13
|
+
def log(message = nil)
|
14
|
+
error message
|
15
|
+
end
|
16
|
+
|
17
|
+
def log_sql(message = nil)
|
18
|
+
say message
|
19
|
+
end
|
20
|
+
|
21
|
+
def abort(message)
|
22
|
+
raise Thor::Error, message
|
23
|
+
end
|
24
|
+
|
25
|
+
# database connection
|
26
|
+
|
27
|
+
def connection
|
28
|
+
@connection ||= begin
|
29
|
+
url = options[:url] || ENV["PGSLICE_URL"]
|
30
|
+
abort "Set PGSLICE_URL or use the --url option" unless url
|
31
|
+
|
32
|
+
uri = URI.parse(url)
|
33
|
+
params = CGI.parse(uri.query.to_s)
|
34
|
+
# remove schema
|
35
|
+
@schema = Array(params.delete("schema") || "public")[0]
|
36
|
+
uri.query = URI.encode_www_form(params)
|
37
|
+
|
38
|
+
ENV["PGCONNECT_TIMEOUT"] ||= "1"
|
39
|
+
PG::Connection.new(uri.to_s)
|
40
|
+
end
|
41
|
+
rescue PG::ConnectionBad => e
|
42
|
+
abort e.message
|
43
|
+
rescue URI::InvalidURIError
|
44
|
+
abort "Invalid url"
|
45
|
+
end
|
46
|
+
|
47
|
+
def schema
|
48
|
+
connection # ensure called first
|
49
|
+
@schema
|
50
|
+
end
|
51
|
+
|
52
|
+
def execute(query, params = [])
|
53
|
+
connection.exec_params(query, params).to_a
|
54
|
+
end
|
55
|
+
|
56
|
+
def run_queries(queries)
|
57
|
+
connection.transaction do
|
58
|
+
execute("SET LOCAL client_min_messages TO warning") unless options[:dry_run]
|
59
|
+
log_sql "BEGIN;"
|
60
|
+
log_sql
|
61
|
+
run_queries_without_transaction(queries)
|
62
|
+
log_sql "COMMIT;"
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def run_query(query)
|
67
|
+
log_sql query
|
68
|
+
unless options[:dry_run]
|
69
|
+
begin
|
70
|
+
execute(query)
|
71
|
+
rescue PG::ServerError => e
|
72
|
+
abort("#{e.class.name}: #{e.message}")
|
73
|
+
end
|
74
|
+
end
|
75
|
+
log_sql
|
76
|
+
end
|
77
|
+
|
78
|
+
def run_queries_without_transaction(queries)
|
79
|
+
queries.each do |query|
|
80
|
+
run_query(query)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def server_version_num
|
85
|
+
execute("SHOW server_version_num")[0]["server_version_num"].to_i
|
86
|
+
end
|
87
|
+
|
88
|
+
# helpers
|
89
|
+
|
90
|
+
def sql_date(time, cast, add_cast = true)
|
91
|
+
if cast == "timestamptz"
|
92
|
+
fmt = "%Y-%m-%d %H:%M:%S UTC"
|
93
|
+
else
|
94
|
+
fmt = "%Y-%m-%d"
|
95
|
+
end
|
96
|
+
str = "'#{time.strftime(fmt)}'"
|
97
|
+
add_cast ? "#{str}::#{cast}" : str
|
98
|
+
end
|
99
|
+
|
100
|
+
def name_format(period)
|
101
|
+
case period.to_sym
|
102
|
+
when :day
|
103
|
+
"%Y%m%d"
|
104
|
+
when :month
|
105
|
+
"%Y%m"
|
106
|
+
else
|
107
|
+
"%Y"
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def partition_date(partition, name_format)
|
112
|
+
DateTime.strptime(partition.name.split("_").last, name_format)
|
113
|
+
end
|
114
|
+
|
115
|
+
def round_date(date, period)
|
116
|
+
date = date.to_date
|
117
|
+
case period.to_sym
|
118
|
+
when :day
|
119
|
+
date
|
120
|
+
when :month
|
121
|
+
Date.new(date.year, date.month)
|
122
|
+
else
|
123
|
+
Date.new(date.year)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def assert_table(table)
|
128
|
+
abort "Table not found: #{table}" unless table.exists?
|
129
|
+
end
|
130
|
+
|
131
|
+
def assert_no_table(table)
|
132
|
+
abort "Table already exists: #{table}" if table.exists?
|
133
|
+
end
|
134
|
+
|
135
|
+
def advance_date(date, period, count = 1)
|
136
|
+
date = date.to_date
|
137
|
+
case period.to_sym
|
138
|
+
when :day
|
139
|
+
date.next_day(count)
|
140
|
+
when :month
|
141
|
+
date.next_month(count)
|
142
|
+
else
|
143
|
+
date.next_year(count)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
def quote_ident(value)
|
148
|
+
PG::Connection.quote_ident(value)
|
149
|
+
end
|
150
|
+
|
151
|
+
def quote_table(table)
|
152
|
+
table.quote_table
|
153
|
+
end
|
154
|
+
|
155
|
+
def quote_no_schema(table)
|
156
|
+
quote_ident(table.name)
|
157
|
+
end
|
158
|
+
|
159
|
+
def create_table(name)
|
160
|
+
if name.include?(".")
|
161
|
+
schema, name = name.split(".", 2)
|
162
|
+
else
|
163
|
+
schema = self.schema
|
164
|
+
end
|
165
|
+
Table.new(schema, name)
|
166
|
+
end
|
167
|
+
|
168
|
+
def make_index_def(index_def, table)
|
169
|
+
index_def.sub(/ ON \S+ USING /, " ON #{quote_table(table)} USING ").sub(/ INDEX .+ ON /, " INDEX ON ") + ";"
|
170
|
+
end
|
171
|
+
|
172
|
+
def make_fk_def(fk_def, table)
|
173
|
+
"ALTER TABLE #{quote_table(table)} ADD #{fk_def};"
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
@@ -0,0 +1,196 @@
|
|
1
|
+
module PgSlice
|
2
|
+
class Table
|
3
|
+
attr_reader :schema, :name
|
4
|
+
|
5
|
+
def initialize(schema, name)
|
6
|
+
@schema = schema
|
7
|
+
@name = name
|
8
|
+
end
|
9
|
+
|
10
|
+
def to_s
|
11
|
+
[schema, name].join(".")
|
12
|
+
end
|
13
|
+
|
14
|
+
def exists?
|
15
|
+
execute("SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE schemaname = $1 AND tablename = $2", [schema, name]).first["count"].to_i > 0
|
16
|
+
end
|
17
|
+
|
18
|
+
def columns
|
19
|
+
execute("SELECT column_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2", [schema, name]).map{ |r| r["column_name"] }
|
20
|
+
end
|
21
|
+
|
22
|
+
# http://www.dbforums.com/showthread.php?1667561-How-to-list-sequences-and-the-columns-by-SQL
|
23
|
+
def sequences
|
24
|
+
query = <<-SQL
|
25
|
+
SELECT
|
26
|
+
a.attname as related_column,
|
27
|
+
s.relname as sequence_name
|
28
|
+
FROM pg_class s
|
29
|
+
JOIN pg_depend d ON d.objid = s.oid
|
30
|
+
JOIN pg_class t ON d.objid = s.oid AND d.refobjid = t.oid
|
31
|
+
JOIN pg_attribute a ON (d.refobjid, d.refobjsubid) = (a.attrelid, a.attnum)
|
32
|
+
JOIN pg_namespace n ON n.oid = s.relnamespace
|
33
|
+
WHERE s.relkind = 'S'
|
34
|
+
AND n.nspname = $1
|
35
|
+
AND t.relname = $2
|
36
|
+
ORDER BY s.relname ASC
|
37
|
+
SQL
|
38
|
+
execute(query, [schema, name])
|
39
|
+
end
|
40
|
+
|
41
|
+
def foreign_keys
|
42
|
+
execute("SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid = #{regclass} AND contype ='f'").map { |r| r["pg_get_constraintdef"] }
|
43
|
+
end
|
44
|
+
|
45
|
+
# https://stackoverflow.com/a/20537829
|
46
|
+
# TODO can simplify with array_position in Postgres 9.5+
|
47
|
+
def primary_key
|
48
|
+
query = <<-SQL
|
49
|
+
SELECT
|
50
|
+
pg_attribute.attname,
|
51
|
+
format_type(pg_attribute.atttypid, pg_attribute.atttypmod),
|
52
|
+
pg_attribute.attnum,
|
53
|
+
pg_index.indkey
|
54
|
+
FROM
|
55
|
+
pg_index, pg_class, pg_attribute, pg_namespace
|
56
|
+
WHERE
|
57
|
+
nspname = $1 AND
|
58
|
+
relname = $2 AND
|
59
|
+
indrelid = pg_class.oid AND
|
60
|
+
pg_class.relnamespace = pg_namespace.oid AND
|
61
|
+
pg_attribute.attrelid = pg_class.oid AND
|
62
|
+
pg_attribute.attnum = any(pg_index.indkey) AND
|
63
|
+
indisprimary
|
64
|
+
SQL
|
65
|
+
rows = execute(query, [schema, name])
|
66
|
+
rows.sort_by { |r| r["indkey"].split(" ").index(r["attnum"]) }.map { |r| r["attname"] }
|
67
|
+
end
|
68
|
+
|
69
|
+
def index_defs
|
70
|
+
execute("SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indrelid = #{regclass} AND indisprimary = 'f'").map { |r| r["pg_get_indexdef"] }
|
71
|
+
end
|
72
|
+
|
73
|
+
def quote_table
|
74
|
+
[quote_ident(schema), quote_ident(name)].join(".")
|
75
|
+
end
|
76
|
+
|
77
|
+
def intermediate_table
|
78
|
+
self.class.new(schema, "#{name}_intermediate")
|
79
|
+
end
|
80
|
+
|
81
|
+
def retired_table
|
82
|
+
self.class.new(schema, "#{name}_retired")
|
83
|
+
end
|
84
|
+
|
85
|
+
def trigger_name
|
86
|
+
"#{name}_insert_trigger"
|
87
|
+
end
|
88
|
+
|
89
|
+
def column_cast(column)
|
90
|
+
data_type = execute("SELECT data_type FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 AND column_name = $3", [schema, name, column])[0]["data_type"]
|
91
|
+
data_type == "timestamp with time zone" ? "timestamptz" : "date"
|
92
|
+
end
|
93
|
+
|
94
|
+
def max_id(primary_key, below: nil, where: nil)
|
95
|
+
query = "SELECT MAX(#{quote_ident(primary_key)}) FROM #{quote_table}"
|
96
|
+
conditions = []
|
97
|
+
conditions << "#{quote_ident(primary_key)} <= #{below}" if below
|
98
|
+
conditions << where if where
|
99
|
+
query << " WHERE #{conditions.join(" AND ")}" if conditions.any?
|
100
|
+
execute(query)[0]["max"].to_i
|
101
|
+
end
|
102
|
+
|
103
|
+
def min_id(primary_key, column, cast, starting_time, where)
|
104
|
+
query = "SELECT MIN(#{quote_ident(primary_key)}) FROM #{quote_table}"
|
105
|
+
conditions = []
|
106
|
+
conditions << "#{quote_ident(column)} >= #{sql_date(starting_time, cast)}" if starting_time
|
107
|
+
conditions << where if where
|
108
|
+
query << " WHERE #{conditions.join(" AND ")}" if conditions.any?
|
109
|
+
(execute(query)[0]["min"] || 1).to_i
|
110
|
+
end
|
111
|
+
|
112
|
+
def partitions
|
113
|
+
query = <<-SQL
|
114
|
+
SELECT
|
115
|
+
nmsp_child.nspname AS schema,
|
116
|
+
child.relname AS name
|
117
|
+
FROM pg_inherits
|
118
|
+
JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
|
119
|
+
JOIN pg_class child ON pg_inherits.inhrelid = child.oid
|
120
|
+
JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace
|
121
|
+
JOIN pg_namespace nmsp_child ON nmsp_child.oid = child.relnamespace
|
122
|
+
WHERE
|
123
|
+
nmsp_parent.nspname = $1 AND
|
124
|
+
parent.relname = $2
|
125
|
+
ORDER BY child.relname ASC
|
126
|
+
SQL
|
127
|
+
execute(query, [schema, name]).map { |r| Table.new(r["schema"], r["name"]) }
|
128
|
+
end
|
129
|
+
|
130
|
+
def fetch_comment
|
131
|
+
execute("SELECT obj_description(#{regclass}) AS comment")[0]
|
132
|
+
end
|
133
|
+
|
134
|
+
def fetch_trigger(trigger_name)
|
135
|
+
execute("SELECT obj_description(oid, 'pg_trigger') AS comment FROM pg_trigger WHERE tgname = $1 AND tgrelid = #{regclass}", [trigger_name])[0]
|
136
|
+
end
|
137
|
+
|
138
|
+
# legacy
|
139
|
+
def fetch_settings(trigger_name)
|
140
|
+
needs_comment = false
|
141
|
+
trigger_comment = fetch_trigger(trigger_name)
|
142
|
+
comment = trigger_comment || fetch_comment
|
143
|
+
if comment
|
144
|
+
field, period, cast, version = comment["comment"].split(",").map { |v| v.split(":").last } rescue []
|
145
|
+
version = version.to_i if version
|
146
|
+
end
|
147
|
+
|
148
|
+
unless period
|
149
|
+
needs_comment = true
|
150
|
+
function_def = execute("SELECT pg_get_functiondef(oid) FROM pg_proc WHERE proname = $1", [trigger_name])[0]
|
151
|
+
return [] unless function_def
|
152
|
+
function_def = function_def["pg_get_functiondef"]
|
153
|
+
sql_format = Helpers::SQL_FORMAT.find { |_, f| function_def.include?("'#{f}'") }
|
154
|
+
return [] unless sql_format
|
155
|
+
period = sql_format[0]
|
156
|
+
field = /to_char\(NEW\.(\w+),/.match(function_def)[1]
|
157
|
+
end
|
158
|
+
|
159
|
+
# backwards compatibility with 0.2.3 and earlier (pre-timestamptz support)
|
160
|
+
unless cast
|
161
|
+
cast = "date"
|
162
|
+
# update comment to explicitly define cast
|
163
|
+
needs_comment = true
|
164
|
+
end
|
165
|
+
|
166
|
+
version ||= trigger_comment ? 1 : 2
|
167
|
+
declarative = version > 1
|
168
|
+
|
169
|
+
[period, field, cast, needs_comment, declarative, version]
|
170
|
+
end
|
171
|
+
|
172
|
+
protected
|
173
|
+
|
174
|
+
def execute(*args)
|
175
|
+
PgSlice::CLI.instance.send(:execute, *args)
|
176
|
+
end
|
177
|
+
|
178
|
+
def quote_ident(value)
|
179
|
+
PG::Connection.quote_ident(value)
|
180
|
+
end
|
181
|
+
|
182
|
+
def regclass
|
183
|
+
"'#{quote_table}'::regclass"
|
184
|
+
end
|
185
|
+
|
186
|
+
def sql_date(time, cast, add_cast = true)
|
187
|
+
if cast == "timestamptz"
|
188
|
+
fmt = "%Y-%m-%d %H:%M:%S UTC"
|
189
|
+
else
|
190
|
+
fmt = "%Y-%m-%d"
|
191
|
+
end
|
192
|
+
str = "'#{time.strftime(fmt)}'"
|
193
|
+
add_cast ? "#{str}::#{cast}" : str
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
data/lib/pgslice/version.rb
CHANGED
metadata
CHANGED
@@ -1,43 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pgslice
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Kane
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-05-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: thor
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: pg
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 0.18.2
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: 0.18.2
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: bundler
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -81,25 +81,28 @@ dependencies:
|
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
description:
|
84
|
-
email:
|
85
|
-
- andrew@chartkick.com
|
84
|
+
email: andrew@chartkick.com
|
86
85
|
executables:
|
87
86
|
- pgslice
|
88
87
|
extensions: []
|
89
88
|
extra_rdoc_files: []
|
90
89
|
files:
|
91
|
-
- ".gitignore"
|
92
|
-
- ".travis.yml"
|
93
90
|
- CHANGELOG.md
|
94
|
-
- Dockerfile
|
95
|
-
- Gemfile
|
96
91
|
- LICENSE.txt
|
97
92
|
- README.md
|
98
|
-
- Rakefile
|
99
93
|
- exe/pgslice
|
100
94
|
- lib/pgslice.rb
|
95
|
+
- lib/pgslice/cli.rb
|
96
|
+
- lib/pgslice/cli/add_partitions.rb
|
97
|
+
- lib/pgslice/cli/analyze.rb
|
98
|
+
- lib/pgslice/cli/fill.rb
|
99
|
+
- lib/pgslice/cli/prep.rb
|
100
|
+
- lib/pgslice/cli/swap.rb
|
101
|
+
- lib/pgslice/cli/unprep.rb
|
102
|
+
- lib/pgslice/cli/unswap.rb
|
103
|
+
- lib/pgslice/helpers.rb
|
104
|
+
- lib/pgslice/table.rb
|
101
105
|
- lib/pgslice/version.rb
|
102
|
-
- pgslice.gemspec
|
103
106
|
homepage: https://github.com/ankane/pgslice
|
104
107
|
licenses:
|
105
108
|
- MIT
|
@@ -112,15 +115,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
112
115
|
requirements:
|
113
116
|
- - ">="
|
114
117
|
- !ruby/object:Gem::Version
|
115
|
-
version: '
|
118
|
+
version: '2.2'
|
116
119
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
117
120
|
requirements:
|
118
121
|
- - ">="
|
119
122
|
- !ruby/object:Gem::Version
|
120
123
|
version: '0'
|
121
124
|
requirements: []
|
122
|
-
|
123
|
-
rubygems_version: 2.7.6
|
125
|
+
rubygems_version: 3.1.2
|
124
126
|
signing_key:
|
125
127
|
specification_version: 4
|
126
128
|
summary: Postgres partitioning as easy as pie
|