pgslice 0.4.1 → 0.4.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,176 @@
1
+ module PgSlice
2
+ module Helpers
3
+ SQL_FORMAT = {
4
+ day: "YYYYMMDD",
5
+ month: "YYYYMM",
6
+ year: "YYYY"
7
+ }
8
+
9
+ protected
10
+
11
+ # output
12
+
13
+ def log(message = nil)
14
+ error message
15
+ end
16
+
17
+ def log_sql(message = nil)
18
+ say message
19
+ end
20
+
21
+ def abort(message)
22
+ raise Thor::Error, message
23
+ end
24
+
25
+ # database connection
26
+
27
+ def connection
28
+ @connection ||= begin
29
+ url = options[:url] || ENV["PGSLICE_URL"]
30
+ abort "Set PGSLICE_URL or use the --url option" unless url
31
+
32
+ uri = URI.parse(url)
33
+ params = CGI.parse(uri.query.to_s)
34
+ # remove schema
35
+ @schema = Array(params.delete("schema") || "public")[0]
36
+ uri.query = URI.encode_www_form(params)
37
+
38
+ ENV["PGCONNECT_TIMEOUT"] ||= "1"
39
+ PG::Connection.new(uri.to_s)
40
+ end
41
+ rescue PG::ConnectionBad => e
42
+ abort e.message
43
+ rescue URI::InvalidURIError
44
+ abort "Invalid url"
45
+ end
46
+
47
+ def schema
48
+ connection # ensure called first
49
+ @schema
50
+ end
51
+
52
+ def execute(query, params = [])
53
+ connection.exec_params(query, params).to_a
54
+ end
55
+
56
+ def run_queries(queries)
57
+ connection.transaction do
58
+ execute("SET LOCAL client_min_messages TO warning") unless options[:dry_run]
59
+ log_sql "BEGIN;"
60
+ log_sql
61
+ run_queries_without_transaction(queries)
62
+ log_sql "COMMIT;"
63
+ end
64
+ end
65
+
66
+ def run_query(query)
67
+ log_sql query
68
+ unless options[:dry_run]
69
+ begin
70
+ execute(query)
71
+ rescue PG::ServerError => e
72
+ abort("#{e.class.name}: #{e.message}")
73
+ end
74
+ end
75
+ log_sql
76
+ end
77
+
78
+ def run_queries_without_transaction(queries)
79
+ queries.each do |query|
80
+ run_query(query)
81
+ end
82
+ end
83
+
84
+ def server_version_num
85
+ execute("SHOW server_version_num")[0]["server_version_num"].to_i
86
+ end
87
+
88
+ # helpers
89
+
90
+ def sql_date(time, cast, add_cast = true)
91
+ if cast == "timestamptz"
92
+ fmt = "%Y-%m-%d %H:%M:%S UTC"
93
+ else
94
+ fmt = "%Y-%m-%d"
95
+ end
96
+ str = "'#{time.strftime(fmt)}'"
97
+ add_cast ? "#{str}::#{cast}" : str
98
+ end
99
+
100
+ def name_format(period)
101
+ case period.to_sym
102
+ when :day
103
+ "%Y%m%d"
104
+ when :month
105
+ "%Y%m"
106
+ else
107
+ "%Y"
108
+ end
109
+ end
110
+
111
+ def partition_date(partition, name_format)
112
+ DateTime.strptime(partition.name.split("_").last, name_format)
113
+ end
114
+
115
+ def round_date(date, period)
116
+ date = date.to_date
117
+ case period.to_sym
118
+ when :day
119
+ date
120
+ when :month
121
+ Date.new(date.year, date.month)
122
+ else
123
+ Date.new(date.year)
124
+ end
125
+ end
126
+
127
+ def assert_table(table)
128
+ abort "Table not found: #{table}" unless table.exists?
129
+ end
130
+
131
+ def assert_no_table(table)
132
+ abort "Table already exists: #{table}" if table.exists?
133
+ end
134
+
135
+ def advance_date(date, period, count = 1)
136
+ date = date.to_date
137
+ case period.to_sym
138
+ when :day
139
+ date.next_day(count)
140
+ when :month
141
+ date.next_month(count)
142
+ else
143
+ date.next_year(count)
144
+ end
145
+ end
146
+
147
+ def quote_ident(value)
148
+ PG::Connection.quote_ident(value)
149
+ end
150
+
151
+ def quote_table(table)
152
+ table.quote_table
153
+ end
154
+
155
+ def quote_no_schema(table)
156
+ quote_ident(table.name)
157
+ end
158
+
159
+ def create_table(name)
160
+ if name.include?(".")
161
+ schema, name = name.split(".", 2)
162
+ else
163
+ schema = self.schema
164
+ end
165
+ Table.new(schema, name)
166
+ end
167
+
168
+ def make_index_def(index_def, table)
169
+ index_def.sub(/ ON \S+ USING /, " ON #{quote_table(table)} USING ").sub(/ INDEX .+ ON /, " INDEX ON ") + ";"
170
+ end
171
+
172
+ def make_fk_def(fk_def, table)
173
+ "ALTER TABLE #{quote_table(table)} ADD #{fk_def};"
174
+ end
175
+ end
176
+ end
@@ -0,0 +1,196 @@
1
+ module PgSlice
2
+ class Table
3
+ attr_reader :schema, :name
4
+
5
+ def initialize(schema, name)
6
+ @schema = schema
7
+ @name = name
8
+ end
9
+
10
+ def to_s
11
+ [schema, name].join(".")
12
+ end
13
+
14
+ def exists?
15
+ execute("SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE schemaname = $1 AND tablename = $2", [schema, name]).first["count"].to_i > 0
16
+ end
17
+
18
+ def columns
19
+ execute("SELECT column_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2", [schema, name]).map{ |r| r["column_name"] }
20
+ end
21
+
22
+ # http://www.dbforums.com/showthread.php?1667561-How-to-list-sequences-and-the-columns-by-SQL
23
+ def sequences
24
+ query = <<-SQL
25
+ SELECT
26
+ a.attname as related_column,
27
+ s.relname as sequence_name
28
+ FROM pg_class s
29
+ JOIN pg_depend d ON d.objid = s.oid
30
+ JOIN pg_class t ON d.objid = s.oid AND d.refobjid = t.oid
31
+ JOIN pg_attribute a ON (d.refobjid, d.refobjsubid) = (a.attrelid, a.attnum)
32
+ JOIN pg_namespace n ON n.oid = s.relnamespace
33
+ WHERE s.relkind = 'S'
34
+ AND n.nspname = $1
35
+ AND t.relname = $2
36
+ ORDER BY s.relname ASC
37
+ SQL
38
+ execute(query, [schema, name])
39
+ end
40
+
41
+ def foreign_keys
42
+ execute("SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid = #{regclass} AND contype ='f'").map { |r| r["pg_get_constraintdef"] }
43
+ end
44
+
45
+ # https://stackoverflow.com/a/20537829
46
+ # TODO can simplify with array_position in Postgres 9.5+
47
+ def primary_key
48
+ query = <<-SQL
49
+ SELECT
50
+ pg_attribute.attname,
51
+ format_type(pg_attribute.atttypid, pg_attribute.atttypmod),
52
+ pg_attribute.attnum,
53
+ pg_index.indkey
54
+ FROM
55
+ pg_index, pg_class, pg_attribute, pg_namespace
56
+ WHERE
57
+ nspname = $1 AND
58
+ relname = $2 AND
59
+ indrelid = pg_class.oid AND
60
+ pg_class.relnamespace = pg_namespace.oid AND
61
+ pg_attribute.attrelid = pg_class.oid AND
62
+ pg_attribute.attnum = any(pg_index.indkey) AND
63
+ indisprimary
64
+ SQL
65
+ rows = execute(query, [schema, name])
66
+ rows.sort_by { |r| r["indkey"].split(" ").index(r["attnum"]) }.map { |r| r["attname"] }
67
+ end
68
+
69
+ def index_defs
70
+ execute("SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indrelid = #{regclass} AND indisprimary = 'f'").map { |r| r["pg_get_indexdef"] }
71
+ end
72
+
73
+ def quote_table
74
+ [quote_ident(schema), quote_ident(name)].join(".")
75
+ end
76
+
77
+ def intermediate_table
78
+ self.class.new(schema, "#{name}_intermediate")
79
+ end
80
+
81
+ def retired_table
82
+ self.class.new(schema, "#{name}_retired")
83
+ end
84
+
85
+ def trigger_name
86
+ "#{name}_insert_trigger"
87
+ end
88
+
89
+ def column_cast(column)
90
+ data_type = execute("SELECT data_type FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 AND column_name = $3", [schema, name, column])[0]["data_type"]
91
+ data_type == "timestamp with time zone" ? "timestamptz" : "date"
92
+ end
93
+
94
+ def max_id(primary_key, below: nil, where: nil)
95
+ query = "SELECT MAX(#{quote_ident(primary_key)}) FROM #{quote_table}"
96
+ conditions = []
97
+ conditions << "#{quote_ident(primary_key)} <= #{below}" if below
98
+ conditions << where if where
99
+ query << " WHERE #{conditions.join(" AND ")}" if conditions.any?
100
+ execute(query)[0]["max"].to_i
101
+ end
102
+
103
+ def min_id(primary_key, column, cast, starting_time, where)
104
+ query = "SELECT MIN(#{quote_ident(primary_key)}) FROM #{quote_table}"
105
+ conditions = []
106
+ conditions << "#{quote_ident(column)} >= #{sql_date(starting_time, cast)}" if starting_time
107
+ conditions << where if where
108
+ query << " WHERE #{conditions.join(" AND ")}" if conditions.any?
109
+ (execute(query)[0]["min"] || 1).to_i
110
+ end
111
+
112
+ def partitions
113
+ query = <<-SQL
114
+ SELECT
115
+ nmsp_child.nspname AS schema,
116
+ child.relname AS name
117
+ FROM pg_inherits
118
+ JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
119
+ JOIN pg_class child ON pg_inherits.inhrelid = child.oid
120
+ JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace
121
+ JOIN pg_namespace nmsp_child ON nmsp_child.oid = child.relnamespace
122
+ WHERE
123
+ nmsp_parent.nspname = $1 AND
124
+ parent.relname = $2
125
+ ORDER BY child.relname ASC
126
+ SQL
127
+ execute(query, [schema, name]).map { |r| Table.new(r["schema"], r["name"]) }
128
+ end
129
+
130
+ def fetch_comment
131
+ execute("SELECT obj_description(#{regclass}) AS comment")[0]
132
+ end
133
+
134
+ def fetch_trigger(trigger_name)
135
+ execute("SELECT obj_description(oid, 'pg_trigger') AS comment FROM pg_trigger WHERE tgname = $1 AND tgrelid = #{regclass}", [trigger_name])[0]
136
+ end
137
+
138
+ # legacy
139
+ def fetch_settings(trigger_name)
140
+ needs_comment = false
141
+ trigger_comment = fetch_trigger(trigger_name)
142
+ comment = trigger_comment || fetch_comment
143
+ if comment
144
+ field, period, cast, version = comment["comment"].split(",").map { |v| v.split(":").last } rescue []
145
+ version = version.to_i if version
146
+ end
147
+
148
+ unless period
149
+ needs_comment = true
150
+ function_def = execute("SELECT pg_get_functiondef(oid) FROM pg_proc WHERE proname = $1", [trigger_name])[0]
151
+ return [] unless function_def
152
+ function_def = function_def["pg_get_functiondef"]
153
+ sql_format = Helpers::SQL_FORMAT.find { |_, f| function_def.include?("'#{f}'") }
154
+ return [] unless sql_format
155
+ period = sql_format[0]
156
+ field = /to_char\(NEW\.(\w+),/.match(function_def)[1]
157
+ end
158
+
159
+ # backwards compatibility with 0.2.3 and earlier (pre-timestamptz support)
160
+ unless cast
161
+ cast = "date"
162
+ # update comment to explicitly define cast
163
+ needs_comment = true
164
+ end
165
+
166
+ version ||= trigger_comment ? 1 : 2
167
+ declarative = version > 1
168
+
169
+ [period, field, cast, needs_comment, declarative, version]
170
+ end
171
+
172
+ protected
173
+
174
+ def execute(*args)
175
+ PgSlice::CLI.instance.send(:execute, *args)
176
+ end
177
+
178
+ def quote_ident(value)
179
+ PG::Connection.quote_ident(value)
180
+ end
181
+
182
+ def regclass
183
+ "'#{quote_table}'::regclass"
184
+ end
185
+
186
+ def sql_date(time, cast, add_cast = true)
187
+ if cast == "timestamptz"
188
+ fmt = "%Y-%m-%d %H:%M:%S UTC"
189
+ else
190
+ fmt = "%Y-%m-%d"
191
+ end
192
+ str = "'#{time.strftime(fmt)}'"
193
+ add_cast ? "#{str}::#{cast}" : str
194
+ end
195
+ end
196
+ end
@@ -1,3 +1,3 @@
1
1
  module PgSlice
2
- VERSION = "0.4.1"
2
+ VERSION = "0.4.6"
3
3
  end
metadata CHANGED
@@ -1,43 +1,43 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pgslice
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.1
4
+ version: 0.4.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Kane
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-05-01 00:00:00.000000000 Z
11
+ date: 2020-05-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: slop
14
+ name: thor
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 4.2.0
19
+ version: '0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: 4.2.0
26
+ version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: pg
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: '0'
33
+ version: 0.18.2
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: '0'
40
+ version: 0.18.2
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: bundler
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -81,25 +81,28 @@ dependencies:
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  description:
84
- email:
85
- - andrew@chartkick.com
84
+ email: andrew@chartkick.com
86
85
  executables:
87
86
  - pgslice
88
87
  extensions: []
89
88
  extra_rdoc_files: []
90
89
  files:
91
- - ".gitignore"
92
- - ".travis.yml"
93
90
  - CHANGELOG.md
94
- - Dockerfile
95
- - Gemfile
96
91
  - LICENSE.txt
97
92
  - README.md
98
- - Rakefile
99
93
  - exe/pgslice
100
94
  - lib/pgslice.rb
95
+ - lib/pgslice/cli.rb
96
+ - lib/pgslice/cli/add_partitions.rb
97
+ - lib/pgslice/cli/analyze.rb
98
+ - lib/pgslice/cli/fill.rb
99
+ - lib/pgslice/cli/prep.rb
100
+ - lib/pgslice/cli/swap.rb
101
+ - lib/pgslice/cli/unprep.rb
102
+ - lib/pgslice/cli/unswap.rb
103
+ - lib/pgslice/helpers.rb
104
+ - lib/pgslice/table.rb
101
105
  - lib/pgslice/version.rb
102
- - pgslice.gemspec
103
106
  homepage: https://github.com/ankane/pgslice
104
107
  licenses:
105
108
  - MIT
@@ -112,15 +115,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
112
115
  requirements:
113
116
  - - ">="
114
117
  - !ruby/object:Gem::Version
115
- version: '0'
118
+ version: '2.2'
116
119
  required_rubygems_version: !ruby/object:Gem::Requirement
117
120
  requirements:
118
121
  - - ">="
119
122
  - !ruby/object:Gem::Version
120
123
  version: '0'
121
124
  requirements: []
122
- rubyforge_project:
123
- rubygems_version: 2.7.6
125
+ rubygems_version: 3.1.2
124
126
  signing_key:
125
127
  specification_version: 4
126
128
  summary: Postgres partitioning as easy as pie