pg_column_byte_packer 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0b3b506f62b7b77b22577939d71057487ac9577573927e31fe9ae88b03e18a24
-  data.tar.gz: 54b478a7824de37ade407f615f25f246014d61ec311087bba86779d71d6cb237
+  metadata.gz: 8ea8169b533cd62fa0b0424f6a45bf27458fe2046bd4788a87733ac768a069dc
+  data.tar.gz: 3a9dd826059c7e874fdd5822fd6f7ca76763632b3ae943e2eb0b609734a9d260
 SHA512:
-  metadata.gz: 8a71b91c4644867ee0462da1d4f08321ddc4aeb409e0910a074ade6fb881338e3445c94c22a0c95fbab21a9c8617f24006404cf262cf8ab1c806076a212a4f05
-  data.tar.gz: 3b752736a44949477bc6d65f41a58901a8ac52d4b8b0637a84d0eeb247b23a2067d39c6a939f7c0a2c69acf0dad5f28066e3e99ff76079dc72f9d87c09de81a4
+  metadata.gz: 7bd2c8c689484c1c55fd4bfc6b7eda5c9f333087b35ba02b6cc697dced20d9769360de66c645b450e295424543c78e1140d5b78b7b59447d95681e1981870d8d
+  data.tar.gz: df2f7e5bc6cc6aa8dd79746bc465854c142ba6ede2e0109021cd1b64bcc216891cd078ab20839950cae668c89ad8d46cad04f53389f2b309a41f496e7c730496

data/.ruby-version

@@ -1 +1 @@
-2.6.0
+ruby-2.5.7

data/README.md

@@ -1,6 +1,6 @@
 # PostgreSQL Column Byte Packer
 
-[![Build Status](https://travis-ci.org/braintree/pg_column_byte_packer.svg?branch=master)](https://travis-ci.org/braintree/pg_column_byte_packer/)
+[![Build Status](https://travis-ci.com/braintree/pg_column_byte_packer.svg?branch=master)](https://travis-ci.org/braintree/pg_column_byte_packer/)
 
 tl;dr: Provides facilities for laying out table column order to optimize for disk space usage both in ActiveRecord migrations and `pg_dump` generated SQL schema files. The general idea and relevant PostgreSQL internals are described in [On Rocks and Sand](https://www.2ndquadrant.com/en/blog/on-rocks-and-sand/).
 
@@ -51,7 +51,7 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 Running tests will automatically create a test database in the locally running Postgres server. You can find the connection parameters in `spec/spec_helper.rb`, but setting the environment variables `PGHOST`, `PGPORT`, `PGUSER`, and `PGPASSWORD` will override the defaults.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). To ignore local changes (say to `.ruby-version`) you can do `rake build release:source_control_push release:rubygem_push`.
 
 ## License
 

data/lib/pg_column_byte_packer.rb

@@ -29,6 +29,37 @@ module PgColumnBytePacker
         end
       end
 
+      if type_schema == "pg_catalog"
+        sql_type = case sql_type
+        when "char", "\"char\""
+          # The SQL standard defines bare 'character' as a single character
+          # type. The quoted variant is also single character, but raises
+          # an error on inputs longer than a single character (unlike the
+          # which requires it to be silently trucated).
+          #
+          # We don't yet use the length for the character type, but it seems
+          # worth retaining it here for completeness (rather than solely
+          # handling the quoted "char" case, which is what we have to have
+          # for things to work minimally).
+          #
+          # See: https://www.postgresql.org/docs/current/datatype-character.html
+          #      and format_type.c
+          "character(1)"
+        when "bit"
+          # The SQL standard defines bare 'bit' as a single bit type.
+          #
+          # See: https://www.postgresql.org/docs/current/datatype-bit.html
+          "bit(1)"
+        when "\"bit\""
+          # The quoted variant, instead of being parallel to the quoted
+          # variant of char, allows any number of bits, but is technically
+          # a different entry in pg_type than varchar.
+          "bit"
+        else
+          sql_type
+        end
+      end
+
       bare_type = if type_schema
         if sql_type.start_with?("#{type_schema}.")
           sql_type.sub("#{type_schema}.", "")
@@ -58,9 +89,16 @@ module PgColumnBytePacker
       # doesn't match the canonical name in pg_type; e.g., "float8" is
       # canonical, but pgdump outputs "double precision".
       case bare_type
-      when "bigint", "double precision", /\Atimestamp.*/, /\Abigserial( primary key)?/
+      when "bigint", "double precision", /\Atimestamp(tz| with time zone| without time zone)?/, /\Abigserial( primary key)?/
         8 # Actual alignment for these types.
-      when "integer", "date", "decimal", "real", /\Aserial( primary key)?/
+      when "integer", "date", /\Atime(tz| with time zone| without time zone)?/, /\Adecimal(\([^\)]+\))?/, "real", /\Aserial( primary key)?/
+        # Note: unlike the others which always take a fixed amount of space,
+        # the numeric/decimal type is stored in a variable amount of space (see:
+        # https://www.postgresql.org/docs/10/datatype-numeric.html) but pg_type
+        # shows that its alignment is the same as integer. Postgres canonicalizes
+        # this type to numeric, but we have to still support the decimal
+        # designation for ActiveRecord inputs.
+
         4 # Actual alignment for these types.
       when "bytea"
         # These types generally have an alignment of 4, but values of at most 127 bytes
@@ -68,20 +106,46 @@ module PgColumnBytePacker
         # Since we'd expect any binary fields to be relatively long, we'll assume they
         # won't fit into the optimized case.
         4
-      when "text", "citext", "character varying"
-        # These types generally have an alignment of 4, but values of at most 127 bytes
-        # long they are optimized into 2 byte alignment.
-        # Since we don't have a good heuristic for determining which columns are likely
-        # to be long or short, we currently just slot them all after the columns we
-        # believe will always be long.
-        # If desired we could also differentiate on length limits if set.
+      when "text", "citext", "character varying", "bit varying", "bit"
+        # These types generally have an alignment of 4 (as designated by pg_type
+        # having a typalign value of 'i', but they're special in that small values
+        # have an optimized storage layout. Beyond the optimized storage layout, though,
+        # these small values also are not required to respect the alignment the type
+        # would otherwise have. Specifically, values with a size of at most 127 bytes
+        # aren't aligned. That 127 byte cap, however, includes an overhead byte to store
+        # the length, and so in reality the max is 126 bytes. Interestingly TOASTable
+        # values are also treated that way, but we don't have a good way of knowing which
+        # values those will be.
+        #
+        # See: `fill_val()` in src/backend/access/common/heaptuple.c (in the conditional
+        # `else if (att->attlen == -1)` branch.
+        #
+        # When no limit modifier has been applied we don't have a good heuristic for
+        # determining which columns are likely to be long or short, so we currently
+        # just slot them all after the columns we believe will always be long.
         3
       when /\Acharacter varying\(\d+\)/
+        # However, when a limit modifier has been applied we can make stronger assumptions.
         if (limit = /\Acharacter varying\((\d+)\)/.match(sql_type)[1])
-          if limit.to_i <= 127
-            2
+          if limit.to_i <= 126
+            # If we know the limit guarantees we'll fit into the unaligned storage
+            # optimization, then we can go ahead and treat it as unaligned.
+            1
           else
-            4
+            # If the limit would allow strings that require alignment, then we assume it's
+            # more likely to exeed the optimization cap and slot them after the columns
+            # we know for certain will require integer alignment.
+            3
+          end
+        end
+      when /\Abit varying\(\d+\)/
+        # This type is functionally the same as varchar above, but the calculation we need
+        # to do has been scaled since the limit is expressed in bits rather than bytes.
+        if (limit = /\Abit varying\((\d+)\)/.match(sql_type)[1])
+          if limit.to_i <= (126 * 8)
+            1
+          else
+            3
           end
         end
       when /\Afloat(\(\d+\))?/
@@ -98,14 +162,30 @@ module PgColumnBytePacker
         else
           8 # Default is double precision
         end
-      when "smallint", "boolean"
-        2 # Actual alignment for these types.
       else
+        type_without_modifier, modifier = bare_type.match(/\A([^\(]+)(\([^\)]+\))?/)[1..-1]
+
+        pg_type_typname = case type_without_modifier
+        when "boolean"
+          "bool"
+        when "smallint"
+          "int2"
+        when "character"
+          "char"
+        else
+          # There are other cases you might expect to see here (for other system
+          # types like varchar and varbit  where the external name generated by
+          # format_type_extended() in Postgres's format_type.c doesn't match the
+          # pg_type typaname), but we've already handled them separately above
+          # (since we have additional rules to apply to them).
+          type_without_modifier
+        end
+
         typtype, typalign = connection.select_rows(<<~SQL, "Type Lookup").first
           SELECT typ.typtype, typ.typalign
           FROM pg_type typ
           JOIN pg_namespace nsp ON nsp.oid = typ.typnamespace
-          WHERE typname = '#{connection.quote_string(bare_type)}'
+          WHERE typname = '#{connection.quote_string(pg_type_typname)}'
             #{type_schema ? "AND nsp.nspname = '#{connection.quote_string(type_schema)}'" : ""}
         SQL
 
@@ -118,7 +198,10 @@ module PgColumnBytePacker
         else
           case typalign
           when "c"
-            0
+            # Character types, for example, occupy a variable amount of space
+            # (though fixed in the sense that it's specified up front for each
+            # column definition) but require no alignment.
+            1
           when "s"
             2
           when "i"
@@ -126,7 +209,7 @@ module PgColumnBytePacker
           when "d"
             8
           else
-            0
+            1
           end
         end
       end

data/lib/pg_column_byte_packer/version.rb

@@ -1,3 +1,3 @@
 module PgColumnBytePacker
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_column_byte_packer
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - jcoleman
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-29 00:00:00.000000000 Z
+date: 2020-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -203,7 +203,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Auto-order table columns for optimize disk space usage