pg_column_byte_packer 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +13 -0
- data/.pryrc +21 -0
- data/.rspec +3 -0
- data/.ruby-version +1 -0
- data/.travis.yml +28 -0
- data/Appraisals +12 -0
- data/Gemfile +6 -0
- data/LICENSE.txt +21 -0
- data/README.md +58 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +9 -0
- data/gemfiles/.bundle/config +2 -0
- data/gemfiles/rails_5.1.gemfile +7 -0
- data/gemfiles/rails_5.2.gemfile +7 -0
- data/gemfiles/rails_6.0.gemfile +7 -0
- data/lib/pg_column_byte_packer.rb +147 -0
- data/lib/pg_column_byte_packer/pg_dump.rb +119 -0
- data/lib/pg_column_byte_packer/schema_statements.rb +44 -0
- data/lib/pg_column_byte_packer/version.rb +3 -0
- data/pg_column_byte_packer.gemspec +37 -0
- metadata +210 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 0b3b506f62b7b77b22577939d71057487ac9577573927e31fe9ae88b03e18a24
|
4
|
+
data.tar.gz: 54b478a7824de37ade407f615f25f246014d61ec311087bba86779d71d6cb237
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8a71b91c4644867ee0462da1d4f08321ddc4aeb409e0910a074ade6fb881338e3445c94c22a0c95fbab21a9c8617f24006404cf262cf8ab1c806076a212a4f05
|
7
|
+
data.tar.gz: 3b752736a44949477bc6d65f41a58901a8ac52d4b8b0637a84d0eeb247b23a2067d39c6a939f7c0a2c69acf0dad5f28066e3e99ff76079dc72f9d87c09de81a4
|
data/.gitignore
ADDED
data/.pryrc
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
if defined?(PryByebug)
|
2
|
+
Pry.commands.alias_command 'c', 'continue'
|
3
|
+
Pry.commands.alias_command 's', 'step'
|
4
|
+
Pry.commands.alias_command 'n', 'next'
|
5
|
+
Pry.commands.alias_command 'f', 'finish'
|
6
|
+
end
|
7
|
+
|
8
|
+
# https://github.com/pry/pry/issues/1275#issuecomment-131969510
|
9
|
+
# Prevent issue where text input does not display on screen in container after typing Ctrl-C in a pry repl
|
10
|
+
at_exit do
|
11
|
+
exit!(1)
|
12
|
+
end
|
13
|
+
|
14
|
+
trap('INT') do
|
15
|
+
begin
|
16
|
+
Pry.run_command "continue", :show_output => true, :target => Pry.current
|
17
|
+
rescue
|
18
|
+
exit
|
19
|
+
end
|
20
|
+
end
|
21
|
+
# End pry Ctrl-C workaround
|
data/.rspec
ADDED
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.6.0
|
data/.travis.yml
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
sudo: false
|
2
|
+
language: ruby
|
3
|
+
rvm:
|
4
|
+
- 2.5
|
5
|
+
env:
|
6
|
+
global:
|
7
|
+
- PGPORT: "5433"
|
8
|
+
jobs:
|
9
|
+
- PGVERSION: "9.6"
|
10
|
+
- PGVERSION: "10"
|
11
|
+
- PGVERSION: "11"
|
12
|
+
- PGVERSION: "12"
|
13
|
+
services:
|
14
|
+
- postgresql
|
15
|
+
before_install:
|
16
|
+
- "for CLUSTER_VERSION in $(pg_lsclusters -h | cut -d' ' -f1); do sudo pg_dropcluster $CLUSTER_VERSION main --stop || true; sudo apt-get remove postgresql-client-$CLUSTER_VERSION postgresql-client-common || true; done"
|
17
|
+
- sudo apt-get update
|
18
|
+
- sudo apt-get -y install postgresql-$PGVERSION postgresql-server-dev-$PGVERSION postgresql-contrib-$PGVERSION postgresql-client-$PGVERSION postgresql-client-common
|
19
|
+
- sudo pg_dropcluster $PGVERSION main --stop || true
|
20
|
+
- sudo pg_createcluster $PGVERSION main -D /var/ramfs/postgresql/11/main -- --auth=trust
|
21
|
+
- sudo pg_ctlcluster start $PGVERSION main
|
22
|
+
- gem uninstall -v '>= 2' -i $(rvm gemdir)@global -ax bundler || true
|
23
|
+
- gem install bundler -v 1.15.4
|
24
|
+
gemfile:
|
25
|
+
- gemfiles/rails_5.1.gemfile
|
26
|
+
- gemfiles/rails_5.2.gemfile
|
27
|
+
- gemfiles/rails_6.0.gemfile
|
28
|
+
script: "bundle exec rake spec"
|
data/Appraisals
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2020 Braintree Payments
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
# PostgreSQL Column Byte Packer
|
2
|
+
|
3
|
+
[](https://travis-ci.org/braintree/pg_column_byte_packer/)
|
4
|
+
|
5
|
+
tl;dr: Provides facilities for laying out table column order to optimize for disk space usage both in ActiveRecord migrations and `pg_dump` generated SQL schema files. The general idea and relevant PostgreSQL internals are described in [On Rocks and Sand](https://www.2ndquadrant.com/en/blog/on-rocks-and-sand/).
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'pg_column_byte_packer'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install pg_column_byte_packer
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
There are two ways you can use this library to byte-pack your tables' column layout:
|
26
|
+
|
27
|
+
### Re-ordering columns using ActiveRecord migrations
|
28
|
+
|
29
|
+
Loading the library automatically patches ActiveRecord's table creation tree walker to automatically re-order columns by alignment size. Therefore all `create_table` calls in ActiveRecord migrations executed after loading the gem will be byte-packing optimized.
|
30
|
+
|
31
|
+
Note: Because you need the full table definition to re-order columns, the most benefit occurs when the full table is created in one step (rather than added onto with repeated `add_column` migrations).
|
32
|
+
|
33
|
+
### Re-ordering SQL structure files (from `pg_dump`)
|
34
|
+
|
35
|
+
ActiveRecord defaults to saving your application's database structure to a `schema.rb` file (which is essentially all of the ActiveRecord migrations commands you'd need to generate the current state of the database). However you can also configure it to save a copy of the database structure in SQL format by setting `config.active_record.schema_format = :sql` in your `config/application.rb` file. With this configuration (and running against a PostgreSQL database) ActiveRecord executes the `pg_dump` utility (included in the PostgreSQL client tools) against your database to generate a `structure.sql` file.
|
36
|
+
|
37
|
+
If you have an existing `structure.sql` file (or any structure-only file generated by `pg_dump`), you can update that file to have byte-packed `CREATE TABLE` statements with the following:
|
38
|
+
|
39
|
+
```ruby
|
40
|
+
PgColumnBytePacker::PgDump.sort_columns_for_definition_file(
|
41
|
+
"<path to structure.sql file>",
|
42
|
+
connection: ActiveRecord::Base.connection
|
43
|
+
)
|
44
|
+
```
|
45
|
+
|
46
|
+
Note: an ActiveRecord connection object is required so that the library can properly determine the data types (and their respective alignment requirements) and column metadata (e.g., `DEFAULT` and `NOT NULL`).
|
47
|
+
|
48
|
+
## Development
|
49
|
+
|
50
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. This project uses Appraisal to test against multiple versions of ActiveRecord; you can run the tests against all supported version with `bundle exec appraisal rspec`.
|
51
|
+
|
52
|
+
Running tests will automatically create a test database in the locally running Postgres server. You can find the connection parameters in `spec/spec_helper.rb`, but setting the environment variables `PGHOST`, `PGPORT`, `PGUSER`, and `PGPASSWORD` will override the defaults.
|
53
|
+
|
54
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
55
|
+
|
56
|
+
## License
|
57
|
+
|
58
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "pg_column_byte_packer"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
@@ -0,0 +1,147 @@
|
|
1
|
+
require "pg_column_byte_packer/version"
|
2
|
+
require "pg_query"
|
3
|
+
|
4
|
+
module PgColumnBytePacker
|
5
|
+
def self.sql_type_alignment_cache
|
6
|
+
@@sql_type_alignment_cache ||= {}
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.ordering_key_for_column(connection:, name:, sql_type:, type_schema: nil, primary_key:, nullable:, has_default:)
|
10
|
+
alignment = PgColumnBytePacker.sql_type_alignment_cache[sql_type] ||= (
|
11
|
+
if type_schema.nil?
|
12
|
+
fake_query = "CREATE TABLE t(c #{sql_type});"
|
13
|
+
parsed = begin
|
14
|
+
PgQuery.parse(fake_query)
|
15
|
+
rescue PgQuery::ParseError
|
16
|
+
nil
|
17
|
+
end
|
18
|
+
|
19
|
+
if parsed && (column_def = parsed.tree[0]["RawStmt"]["stmt"]["CreateStmt"]["tableElts"][0]["ColumnDef"])
|
20
|
+
type_identifiers = column_def["typeName"]["TypeName"]["names"].map { |s| s["String"]["str"] }
|
21
|
+
case type_identifiers.size
|
22
|
+
when 1
|
23
|
+
# Do nothing; we already have a bare type.
|
24
|
+
when 2
|
25
|
+
type_schema, bare_type = type_identifiers
|
26
|
+
else
|
27
|
+
raise ArgumentError, "Unexpected number of identifiers in type declaration for column: `#{name}`, identifiers: #{type_identifiers.inspect}"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
bare_type = if type_schema
|
33
|
+
if sql_type.start_with?("#{type_schema}.")
|
34
|
+
sql_type.sub("#{type_schema}.", "")
|
35
|
+
elsif sql_type.start_with?("\"#{type_schema}\".")
|
36
|
+
sql_type.sub("\"#{type_schema}\".", "")
|
37
|
+
else
|
38
|
+
sql_type
|
39
|
+
end
|
40
|
+
else
|
41
|
+
sql_type
|
42
|
+
end
|
43
|
+
|
44
|
+
# Ignore array designations. This seems like something we could
|
45
|
+
# so with the parsing above, and we could, and, in fact, that
|
46
|
+
# would also almost certain allow us to rip out most of the
|
47
|
+
# ActiveRecord generate alias type name matching below, but
|
48
|
+
# it would also mean a more thorough refactor below for types
|
49
|
+
# with size designations (e.g., when ActiveRecord generates
|
50
|
+
# "float(23)"). So we use this simple regex cleanup for now.
|
51
|
+
bare_type = bare_type.sub(/(\[\])+\Z/, "")
|
52
|
+
|
53
|
+
# Sort out the alignment. Most of the type name matching is to
|
54
|
+
# support the naming variants that ActiveRecord generates (often
|
55
|
+
# they're aliases, like "integer", for which PostgreSQL internally
|
56
|
+
# has a different canonical name, like "int4"). There are also
|
57
|
+
# a few cases we have to handle where the output from pgdump
|
58
|
+
# doesn't match the canonical name in pg_type; e.g., "float8" is
|
59
|
+
# canonical, but pgdump outputs "double precision".
|
60
|
+
case bare_type
|
61
|
+
when "bigint", "double precision", /\Atimestamp.*/, /\Abigserial( primary key)?/
|
62
|
+
8 # Actual alignment for these types.
|
63
|
+
when "integer", "date", "decimal", "real", /\Aserial( primary key)?/
|
64
|
+
4 # Actual alignment for these types.
|
65
|
+
when "bytea"
|
66
|
+
# These types generally have an alignment of 4, but values of at most 127 bytes
|
67
|
+
# long they are optimized into 2 byte alignment.
|
68
|
+
# Since we'd expect any binary fields to be relatively long, we'll assume they
|
69
|
+
# won't fit into the optimized case.
|
70
|
+
4
|
71
|
+
when "text", "citext", "character varying"
|
72
|
+
# These types generally have an alignment of 4, but values of at most 127 bytes
|
73
|
+
# long they are optimized into 2 byte alignment.
|
74
|
+
# Since we don't have a good heuristic for determining which columns are likely
|
75
|
+
# to be long or short, we currently just slot them all after the columns we
|
76
|
+
# believe will always be long.
|
77
|
+
# If desired we could also differentiate on length limits if set.
|
78
|
+
3
|
79
|
+
when /\Acharacter varying\(\d+\)/
|
80
|
+
if (limit = /\Acharacter varying\((\d+)\)/.match(sql_type)[1])
|
81
|
+
if limit.to_i <= 127
|
82
|
+
2
|
83
|
+
else
|
84
|
+
4
|
85
|
+
end
|
86
|
+
end
|
87
|
+
when /\Afloat(\(\d+\))?/
|
88
|
+
precision_match = /\Afloat\((\d+)\)?/.match(sql_type)
|
89
|
+
if precision_match
|
90
|
+
# Precision here is a number of binary digits;
|
91
|
+
# see https://www.postgresql.org/docs/10/datatype-numeric.html
|
92
|
+
# for more information.
|
93
|
+
if precision_match[1].to_i >= 25
|
94
|
+
8 # Double precision
|
95
|
+
else
|
96
|
+
4 # Real
|
97
|
+
end
|
98
|
+
else
|
99
|
+
8 # Default is double precision
|
100
|
+
end
|
101
|
+
when "smallint", "boolean"
|
102
|
+
2 # Actual alignment for these types.
|
103
|
+
else
|
104
|
+
typtype, typalign = connection.select_rows(<<~SQL, "Type Lookup").first
|
105
|
+
SELECT typ.typtype, typ.typalign
|
106
|
+
FROM pg_type typ
|
107
|
+
JOIN pg_namespace nsp ON nsp.oid = typ.typnamespace
|
108
|
+
WHERE typname = '#{connection.quote_string(bare_type)}'
|
109
|
+
#{type_schema ? "AND nsp.nspname = '#{connection.quote_string(type_schema)}'" : ""}
|
110
|
+
SQL
|
111
|
+
|
112
|
+
if typtype.nil?
|
113
|
+
raise ArgumentError, "Got sql_type: `#{sql_type}` and type_schema: `#{type_schema}` but was unable to find entry in pg_type."
|
114
|
+
end
|
115
|
+
|
116
|
+
if typtype == "e"
|
117
|
+
4
|
118
|
+
else
|
119
|
+
case typalign
|
120
|
+
when "c"
|
121
|
+
0
|
122
|
+
when "s"
|
123
|
+
2
|
124
|
+
when "i"
|
125
|
+
4
|
126
|
+
when "d"
|
127
|
+
8
|
128
|
+
else
|
129
|
+
0
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
)
|
134
|
+
|
135
|
+
# Ordering components in order of most importance to least importance.
|
136
|
+
[
|
137
|
+
-alignment, # Sort alignment descending.
|
138
|
+
primary_key ? 0 : 1, # Sort PRIMARY KEY first.
|
139
|
+
nullable ? 1 : 0, # Sort NOT NULL first.
|
140
|
+
has_default && nullable ? 0 : 1, # Sort DEFAULT first (but only when also nullable).
|
141
|
+
name, # Sort name ascending.
|
142
|
+
]
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
require "pg_column_byte_packer/schema_statements"
|
147
|
+
require "pg_column_byte_packer/pg_dump"
|
@@ -0,0 +1,119 @@
|
|
1
|
+
module PgColumnBytePacker
|
2
|
+
module PgDump
|
3
|
+
def self.sort_columns_for_definition_file(path, connection:)
|
4
|
+
sorted_dump_output = []
|
5
|
+
current_block_start = nil
|
6
|
+
current_block_body = []
|
7
|
+
current_table_name = nil
|
8
|
+
block_begin_prefix_pattern = /\ACREATE TABLE ([^\(]+) \(/
|
9
|
+
block_end_line_pattern = /\A\);?\n\Z/
|
10
|
+
|
11
|
+
File.foreach(path) do |line|
|
12
|
+
if current_block_start.nil? && (create_table_match = block_begin_prefix_pattern.match(line))
|
13
|
+
current_block_start = line
|
14
|
+
current_block_body = []
|
15
|
+
current_table_name = create_table_match[1]
|
16
|
+
elsif current_block_start
|
17
|
+
if line =~ block_end_line_pattern
|
18
|
+
sorted_dump_output << current_block_start
|
19
|
+
table_lines = _sort_table_lines(
|
20
|
+
connection: connection,
|
21
|
+
qualified_table: current_table_name,
|
22
|
+
lines: current_block_body
|
23
|
+
)
|
24
|
+
table_lines = table_lines.map.with_index do |column_line, index|
|
25
|
+
has_trailing_comma = column_line =~ /,\s*\Z/
|
26
|
+
last_line = index == table_lines.size - 1
|
27
|
+
if !last_line && !has_trailing_comma
|
28
|
+
column_line.sub(/(.+)(\s*)\Z/, '\1,\2')
|
29
|
+
elsif last_line && has_trailing_comma
|
30
|
+
column_line.sub(/,(\s*)\Z/, '\1')
|
31
|
+
else
|
32
|
+
column_line
|
33
|
+
end
|
34
|
+
end
|
35
|
+
sorted_dump_output.concat(table_lines)
|
36
|
+
sorted_dump_output << line
|
37
|
+
|
38
|
+
current_block_body = []
|
39
|
+
current_block_start = nil
|
40
|
+
current_table_name = nil
|
41
|
+
else
|
42
|
+
current_block_body << line
|
43
|
+
end
|
44
|
+
else
|
45
|
+
sorted_dump_output << line
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
File.write(path, sorted_dump_output.join)
|
50
|
+
end
|
51
|
+
|
52
|
+
def self._sort_table_lines(connection:, qualified_table:, lines:)
|
53
|
+
schema, table = qualified_table.match(/([^\.]+)\.([^\.]+)/)[1..-1]
|
54
|
+
|
55
|
+
lines.sort_by do |line|
|
56
|
+
line = line.chomp
|
57
|
+
|
58
|
+
# To handle the vagaries of quoted keywords as column names
|
59
|
+
# and spaces/special characters in column names, we resort
|
60
|
+
# to using PostgreSQL's parsing code itself on a faked
|
61
|
+
# CREATE TABLE call with a single column; we could parse
|
62
|
+
# the entire CREATE TABLE statement we have in the file,
|
63
|
+
# but then we'd have to figure out a way to recreate that
|
64
|
+
# query from the parse tree, and therein lies madness.
|
65
|
+
line_without_comma = line[-1] == "," ? line[0..-2] : line
|
66
|
+
fake_query = "CREATE TABLE t(#{line_without_comma});"
|
67
|
+
parsed = begin
|
68
|
+
PgQuery.parse(fake_query)
|
69
|
+
rescue PgQuery::ParseError
|
70
|
+
nil
|
71
|
+
end
|
72
|
+
|
73
|
+
# Ignore CONSTRAINT definitions etc.
|
74
|
+
if parsed && (column_def = parsed.tree[0]["RawStmt"]["stmt"]["CreateStmt"]["tableElts"][0]["ColumnDef"])
|
75
|
+
column = column_def["colname"]
|
76
|
+
|
77
|
+
values = connection.select_rows(<<~SQL, "Column and Type Info").first
|
78
|
+
SELECT
|
79
|
+
pg_catalog.format_type(attr.atttypid, attr.atttypmod),
|
80
|
+
typ_nsp.nspname,
|
81
|
+
attr.attnotnull,
|
82
|
+
attr.atthasdef,
|
83
|
+
EXISTS (
|
84
|
+
SELECT 1
|
85
|
+
FROM pg_index idx
|
86
|
+
WHERE idx.indisprimary
|
87
|
+
AND attr.attnum = ANY(idx.indkey)
|
88
|
+
AND idx.indrelid = attr.attrelid
|
89
|
+
)
|
90
|
+
FROM pg_catalog.pg_attribute attr
|
91
|
+
JOIN pg_catalog.pg_type typ ON typ.oid = attr.atttypid
|
92
|
+
JOIN pg_catalog.pg_class cls ON cls.oid = attr.attrelid
|
93
|
+
JOIN pg_catalog.pg_namespace attr_nsp ON attr_nsp.oid = cls.relnamespace
|
94
|
+
JOIN pg_catalog.pg_namespace typ_nsp ON typ_nsp.oid = typ.typnamespace
|
95
|
+
WHERE attr.attname = '#{connection.quote_string(column)}'
|
96
|
+
AND attr_nsp.nspname = '#{connection.quote_string(schema)}'
|
97
|
+
AND cls.relname = '#{connection.quote_string(table)}'
|
98
|
+
AND NOT attr.attisdropped
|
99
|
+
SQL
|
100
|
+
sql_type, type_schema, not_null, has_default, primary_key = values
|
101
|
+
|
102
|
+
PgColumnBytePacker.ordering_key_for_column(
|
103
|
+
connection: connection,
|
104
|
+
name: column,
|
105
|
+
sql_type: sql_type,
|
106
|
+
type_schema: type_schema,
|
107
|
+
primary_key: primary_key,
|
108
|
+
nullable: !not_null,
|
109
|
+
has_default: has_default
|
110
|
+
)
|
111
|
+
else
|
112
|
+
# All non-column lines we want to sort at the end
|
113
|
+
# of the table defintion statement.
|
114
|
+
[2**32]
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require "active_record"
|
2
|
+
require "active_record/migration"
|
3
|
+
|
4
|
+
module PgColumnBytePacker
|
5
|
+
module SchemaCreation
|
6
|
+
def visit_TableDefinition(o)
|
7
|
+
columns_hash = o.instance_variable_get(:@columns_hash)
|
8
|
+
|
9
|
+
sorted_column_tuples = columns_hash.sort_by do |name, col|
|
10
|
+
sql_type = type_to_sql(
|
11
|
+
col.type,
|
12
|
+
:limit => col.limit,
|
13
|
+
:precision => col.precision,
|
14
|
+
:scale => col.scale,
|
15
|
+
:primary_key => col.primary_key?,
|
16
|
+
)
|
17
|
+
|
18
|
+
nullable = if sql_type.match(/\A(big)?serial( primary key)?/)
|
19
|
+
col.null == true
|
20
|
+
else
|
21
|
+
col.null.nil? || col.null == true
|
22
|
+
end
|
23
|
+
|
24
|
+
PgColumnBytePacker.ordering_key_for_column(
|
25
|
+
connection: @conn,
|
26
|
+
name: name,
|
27
|
+
sql_type: sql_type,
|
28
|
+
primary_key: col.options[:primary_key],
|
29
|
+
nullable: nullable,
|
30
|
+
has_default: !col.default.nil?
|
31
|
+
)
|
32
|
+
end
|
33
|
+
|
34
|
+
columns_hash.clear
|
35
|
+
sorted_column_tuples.each do |(name, column)|
|
36
|
+
columns_hash[name] = column
|
37
|
+
end
|
38
|
+
|
39
|
+
super(o)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
ActiveRecord::ConnectionAdapters::AbstractAdapter::SchemaCreation.prepend(PgColumnBytePacker::SchemaCreation)
|
@@ -0,0 +1,37 @@
|
|
1
|
+
lib = File.expand_path("../lib", __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require "pg_column_byte_packer/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "pg_column_byte_packer"
|
7
|
+
spec.version = PgColumnBytePacker::VERSION
|
8
|
+
spec.authors = [
|
9
|
+
"jcoleman",
|
10
|
+
]
|
11
|
+
spec.email = ["code@getbraintree.com"]
|
12
|
+
|
13
|
+
spec.summary = %q{Auto-order table columns for optimize disk space usage}
|
14
|
+
spec.homepage = "https://github.com/braintree/pg_column_byte_packer"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
# Specify which files should be added to the gem when it is released.
|
18
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
19
|
+
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
20
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
21
|
+
end
|
22
|
+
spec.bindir = "exe"
|
23
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
24
|
+
spec.require_paths = ["lib"]
|
25
|
+
|
26
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
27
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
28
|
+
spec.add_development_dependency "relation_to_struct"
|
29
|
+
spec.add_development_dependency "db-query-matchers", "~> 0.9.0"
|
30
|
+
spec.add_development_dependency "pry"
|
31
|
+
spec.add_development_dependency "pry-byebug"
|
32
|
+
spec.add_development_dependency "appraisal", "~> 2.2.0"
|
33
|
+
|
34
|
+
spec.add_dependency "pg"
|
35
|
+
spec.add_dependency "activerecord", ">= 5.1", "< 6.1"
|
36
|
+
spec.add_dependency "pg_query"
|
37
|
+
end
|
metadata
ADDED
@@ -0,0 +1,210 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: pg_column_byte_packer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- jcoleman
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-09-29 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '10.0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '10.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rspec
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '3.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '3.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: relation_to_struct
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: db-query-matchers
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.9.0
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.9.0
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pry
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: pry-byebug
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: appraisal
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 2.2.0
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 2.2.0
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: pg
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: activerecord
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '5.1'
|
132
|
+
- - "<"
|
133
|
+
- !ruby/object:Gem::Version
|
134
|
+
version: '6.1'
|
135
|
+
type: :runtime
|
136
|
+
prerelease: false
|
137
|
+
version_requirements: !ruby/object:Gem::Requirement
|
138
|
+
requirements:
|
139
|
+
- - ">="
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '5.1'
|
142
|
+
- - "<"
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: '6.1'
|
145
|
+
- !ruby/object:Gem::Dependency
|
146
|
+
name: pg_query
|
147
|
+
requirement: !ruby/object:Gem::Requirement
|
148
|
+
requirements:
|
149
|
+
- - ">="
|
150
|
+
- !ruby/object:Gem::Version
|
151
|
+
version: '0'
|
152
|
+
type: :runtime
|
153
|
+
prerelease: false
|
154
|
+
version_requirements: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - ">="
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '0'
|
159
|
+
description:
|
160
|
+
email:
|
161
|
+
- code@getbraintree.com
|
162
|
+
executables: []
|
163
|
+
extensions: []
|
164
|
+
extra_rdoc_files: []
|
165
|
+
files:
|
166
|
+
- ".gitignore"
|
167
|
+
- ".pryrc"
|
168
|
+
- ".rspec"
|
169
|
+
- ".ruby-version"
|
170
|
+
- ".travis.yml"
|
171
|
+
- Appraisals
|
172
|
+
- Gemfile
|
173
|
+
- LICENSE.txt
|
174
|
+
- README.md
|
175
|
+
- Rakefile
|
176
|
+
- bin/console
|
177
|
+
- bin/setup
|
178
|
+
- gemfiles/.bundle/config
|
179
|
+
- gemfiles/rails_5.1.gemfile
|
180
|
+
- gemfiles/rails_5.2.gemfile
|
181
|
+
- gemfiles/rails_6.0.gemfile
|
182
|
+
- lib/pg_column_byte_packer.rb
|
183
|
+
- lib/pg_column_byte_packer/pg_dump.rb
|
184
|
+
- lib/pg_column_byte_packer/schema_statements.rb
|
185
|
+
- lib/pg_column_byte_packer/version.rb
|
186
|
+
- pg_column_byte_packer.gemspec
|
187
|
+
homepage: https://github.com/braintree/pg_column_byte_packer
|
188
|
+
licenses:
|
189
|
+
- MIT
|
190
|
+
metadata: {}
|
191
|
+
post_install_message:
|
192
|
+
rdoc_options: []
|
193
|
+
require_paths:
|
194
|
+
- lib
|
195
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
196
|
+
requirements:
|
197
|
+
- - ">="
|
198
|
+
- !ruby/object:Gem::Version
|
199
|
+
version: '0'
|
200
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
201
|
+
requirements:
|
202
|
+
- - ">="
|
203
|
+
- !ruby/object:Gem::Version
|
204
|
+
version: '0'
|
205
|
+
requirements: []
|
206
|
+
rubygems_version: 3.1.2
|
207
|
+
signing_key:
|
208
|
+
specification_version: 4
|
209
|
+
summary: Auto-order table columns for optimize disk space usage
|
210
|
+
test_files: []
|