pgslice 0.4.8 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4d3a279841af6ee62759c9570149cda73460c674c383aef78d7b99fa5c9840ec
-  data.tar.gz: 48462eb02783c097d0553071880e75650263d662cf755e8fa977b1d4e25ccc0e
+  metadata.gz: 2f56f37e43ecc30fd145316bde2bd20478b360c077f928323fdee982379433b8
+  data.tar.gz: 594dc56e38e2d5902df435ddc514cdbb15004432f4b2b0d5d95ae37c7f8b00a5
 SHA512:
-  metadata.gz: 9c95fe04d9c90f524c9cd778f5a5e05013ff4655982cd4ed1420a024c5c17a97d3de679c86e1d66800a9ee3ab7e29eec8abec8676041d7e07d7ebce96052315f
-  data.tar.gz: 5d43e4d5dc94f29538abd25bd2e3676dc47e1b5cdce646e345539d7543f4b21fb2d9d33ecb08c69df81283150643ad423817179719f712d963b0aba791c10d89
+  metadata.gz: 4cda3774db98a20ba3d13e1655baa7bcd3f294f810457d1c1a2a6ff7b3da1e861814d26638a95fcfa8f9fcad3280b0a6e419fead798cf83fb76e30108bd8b567
+  data.tar.gz: 62b2bc217e0fc0e822e4852159790b5d0e7d5e20a8d27216890362975de7100253183834c383b618c2d7ed1623891fbd41f76d44145e3274d6f7b0e1a1a7a225

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.6.0 (2023-04-22)
+
+- Added support for generated columns
+- Added compression and extended statistics to `prep`
+- Dropped support for Ruby < 2.7
+- Dropped support for Postgres < 11
+
+## 0.5.0 (2023-01-29)
+
+- Dropped support for Ruby < 2.5
+
 ## 0.4.8 (2022-02-28)
 
 - Fixed error with pg 1.3

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016-2022 Andrew Kane
+Copyright (c) 2016-2023 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -14,7 +14,7 @@ pgslice is a command line tool. To install, run:
 gem install pgslice
 ```
 
-This will give you the `pgslice` command. You can also install it with [Homebrew](#homebrew). If installation fails, you may need to install [dependencies](#dependencies).
+This will give you the `pgslice` command. You can also install it with [Homebrew](#homebrew) or [Docker](#docker). If installation fails, you may need to install [dependencies](#dependencies).
 
 ## Steps
 
@@ -32,9 +32,9 @@ This will give you the `pgslice` command. You can also install it with [Homebrew
   pgslice prep <table> <column> <period>
   ```
 
-  Period can be `day`, `month`, or `year`.
+  The column should be a `timestamp`, `timestamptz`, or `date` column and period can be `day`, `month`, or `year`.
 
-  This creates a partitioned table named `<table>_intermediate`.
+  This creates a partitioned table named `<table>_intermediate` using range partitioning.
 
 4. Add partitions to the intermediate table
 
@@ -50,7 +50,7 @@ This will give you the `pgslice` command. You can also install it with [Homebrew
   pgslice fill <table>
   ```
 
-  Use the `--batch-size` and `--sleep` options to control the speed.
+  Use the `--batch-size` and `--sleep` options to control the speed (defaults to `10000` and `0` respectively)
 
   To sync data across different databases, check out [pgsync](https://github.com/ankane/pgsync).
 
@@ -108,17 +108,17 @@ pgslice add_partitions visits --intermediate --past 1 --future 1
 ```sql
 BEGIN;
 
-CREATE TABLE "public"."visits_201808" PARTITION OF "public"."visits_intermediate" FOR VALUES FROM ('2018-08-01') TO ('2018-09-01');
+CREATE TABLE "public"."visits_202208" PARTITION OF "public"."visits_intermediate" FOR VALUES FROM ('2022-08-01') TO ('2022-09-01');
 
-ALTER TABLE "public"."visits_201808" ADD PRIMARY KEY ("id");
+ALTER TABLE "public"."visits_202208" ADD PRIMARY KEY ("id");
 
-CREATE TABLE "public"."visits_201809" PARTITION OF "public"."visits_intermediate" FOR VALUES FROM ('2018-09-01') TO ('2018-10-01');
+CREATE TABLE "public"."visits_202209" PARTITION OF "public"."visits_intermediate" FOR VALUES FROM ('2022-09-01') TO ('2022-10-01');
 
-ALTER TABLE "public"."visits_201809" ADD PRIMARY KEY ("id");
+ALTER TABLE "public"."visits_202209" ADD PRIMARY KEY ("id");
 
-CREATE TABLE "public"."visits_201810" PARTITION OF "public"."visits_intermediate" FOR VALUES FROM ('2018-10-01') TO ('2018-11-01');
+CREATE TABLE "public"."visits_202210" PARTITION OF "public"."visits_intermediate" FOR VALUES FROM ('2022-10-01') TO ('2022-11-01');
 
-ALTER TABLE "public"."visits_201808" ADD PRIMARY KEY ("id");
+ALTER TABLE "public"."visits_202210" ADD PRIMARY KEY ("id");
 
 COMMIT;
 ```
@@ -131,17 +131,17 @@ pgslice fill visits
 /* 1 of 3 */
 INSERT INTO "public"."visits_intermediate" ("id", "user_id", "ip", "created_at")
     SELECT "id", "user_id", "ip", "created_at" FROM "public"."visits"
-    WHERE "id" > 0 AND "id" <= 10000 AND "created_at" >= '2018-08-01'::date AND "created_at" < '2018-11-01'::date
+    WHERE "id" > 0 AND "id" <= 10000 AND "created_at" >= '2022-08-01'::date AND "created_at" < '2022-11-01'::date
 
 /* 2 of 3 */
 INSERT INTO "public"."visits_intermediate" ("id", "user_id", "ip", "created_at")
     SELECT "id", "user_id", "ip", "created_at" FROM "public"."visits"
-    WHERE "id" > 10000 AND "id" <= 20000 AND "created_at" >= '2018-08-01'::date AND "created_at" < '2018-11-01'::date
+    WHERE "id" > 10000 AND "id" <= 20000 AND "created_at" >= '2022-08-01'::date AND "created_at" < '2022-11-01'::date
 
 /* 3 of 3 */
 INSERT INTO "public"."visits_intermediate" ("id", "user_id", "ip", "created_at")
     SELECT "id", "user_id", "ip", "created_at" FROM "public"."visits"
-    WHERE "id" > 20000 AND "id" <= 30000 AND "created_at" >= '2018-08-01'::date AND "created_at" < '2018-11-01'::date
+    WHERE "id" > 20000 AND "id" <= 30000 AND "created_at" >= '2022-08-01'::date AND "created_at" < '2022-11-01'::date
 ```
 
 ```sh
@@ -149,11 +149,11 @@ pgslice analyze visits
 ```
 
 ```sql
-ANALYZE VERBOSE "public"."visits_201808";
+ANALYZE VERBOSE "public"."visits_202208";
 
-ANALYZE VERBOSE "public"."visits_201809";
+ANALYZE VERBOSE "public"."visits_202209";
 
-ANALYZE VERBOSE "public"."visits_201810";
+ANALYZE VERBOSE "public"."visits_202210";
 
 ANALYZE VERBOSE "public"."visits_intermediate";
 ```
@@ -217,25 +217,20 @@ WHERE
 Back up and drop older partitions each day, month, or year.
 
 ```sh
-pg_dump -c -Fc -t <table>_201809 $PGSLICE_URL > <table>_201809.dump
-psql -c "DROP TABLE <table>_201809" $PGSLICE_URL
+pg_dump -c -Fc -t <table>_202209 $PGSLICE_URL > <table>_202209.dump
+psql -c "DROP TABLE <table>_202209" $PGSLICE_URL
 ```
 
 If you use [Amazon S3](https://aws.amazon.com/s3/) for backups, [s3cmd](https://github.com/s3tools/s3cmd) is a nice tool.
 
 ```sh
-s3cmd put <table>_201809.dump s3://<s3-bucket>/<table>_201809.dump
+s3cmd put <table>_202209.dump s3://<s3-bucket>/<table>_202209.dump
 ```
 
 ## Schema Updates
 
 Once a table is partitioned, make schema updates on the master table only (not partitions). This includes adding, removing, and modifying columns, as well as adding and removing indexes and foreign keys.
 
-A few exceptions are:
-
-- For Postgres 10, make index and foreign key updates on partitions only
-- For Postgres < 10, make index and foreign key updates on the master table and all partitions
-
 ## Additional Commands
 
 To undo prep (which will delete partitions), use:
@@ -272,27 +267,20 @@ SELECT * FROM
 WHERE
     user_id = 123 AND
     -- for performance
-    created_at >= '2018-09-01' AND created_at < '2018-09-02'
+    created_at >= '2022-09-01' AND created_at < '2022-09-02'
 ```
 
-For this to be effective, ensure `constraint_exclusion` is set to `partition` (default value) or `on`.
+For this to be effective, ensure `constraint_exclusion` is set to `partition` (the default value) or `on`.
 
 ```sql
 SHOW constraint_exclusion;
 ```
 
-### Writes
-
-Before Postgres 10, if you use `INSERT` statements with a `RETURNING` clause (as frameworks like Rails do), you’ll no longer receive the id of the newly inserted record(s) back. If you need this, you can either:
-
-1. Insert directly into the partition
-2. Get value before the insert with `SELECT nextval('sequence_name')` (for multiple rows, append `FROM generate_series(1, n)`)
-
 ## Frameworks
 
 ### Rails
 
-For Postgres 10+, specify the primary key for partitioned models to ensure it’s returned.
+Specify the primary key for partitioned models to ensure it’s returned.
 
 ```ruby
 class Visit < ApplicationRecord
@@ -300,16 +288,6 @@ class Visit < ApplicationRecord
 end
 ```
 
-Before Postgres 10, preload the value.
-
-```ruby
-class Visit < ApplicationRecord
-  before_create do
-    self.id ||= self.class.connection.select_all("SELECT nextval('#{self.class.sequence_name}')").first["nextval"]
-  end
-end
-```
-
 ### Other Frameworks
 
 Please submit a PR if additional configuration is needed.
@@ -328,22 +306,31 @@ pgslice swap <table>
 
 Triggers aren’t copied from the original table. You can set up triggers on the intermediate table if needed. Note that Postgres doesn’t support `BEFORE / FOR EACH ROW` triggers on partitioned tables.
 
-## Declarative Partitioning
-
-Postgres 10 introduces [declarative partitioning](https://www.postgresql.org/docs/10/static/ddl-partitioning.html#ddl-partitioning-declarative). A major benefit is `INSERT` statements with a `RETURNING` clause work as expected. If you prefer to use trigger-based partitioning instead (not recommended), pass the `--trigger-based` option to the `prep` command.
-
 ## Data Protection
 
 Always make sure your [connection is secure](https://ankane.org/postgres-sslmode-explained) when connecting to a database over a network you don’t fully trust. Your best option is to connect over SSH or a VPN. Another option is to use `sslmode=verify-full`. If you don’t do this, your database credentials can be compromised.
 
-## Homebrew
+## Additional Installation Methods
+
+### Homebrew
 
-On Mac, you can use:
+With Homebrew, you can use:
 
 ```sh
 brew install ankane/brew/pgslice
 ```
 
+### Docker
+
+Get the [Docker image](https://hub.docker.com/r/ankane/pgslice) with:
+
+```sh
+docker pull ankane/pgslice
+alias pgslice="docker run --rm -e PGSLICE_URL ankane/pgslice"
+```
+
+This will give you the `pgslice` command.
+
 ## Dependencies
 
 If installation fails, your system may be missing Ruby or libpq.
@@ -351,7 +338,7 @@ If installation fails, your system may be missing Ruby or libpq.
 On Mac, run:
 
 ```sh
-brew install postgresql
+brew install libpq
 ```
 
 On Ubuntu, run:
@@ -375,15 +362,6 @@ gem install specific_install
 gem specific_install https://github.com/ankane/pgslice.git
 ```
 
-## Docker
-
-```sh
-docker build -t pgslice .
-alias pgslice="docker run --rm -e PGSLICE_URL pgslice"
-```
-
-This will give you the `pgslice` command.
-
 ## Reference
 
 - [PostgreSQL Manual](https://www.postgresql.org/docs/current/static/ddl-partitioning.html)

data/lib/pgslice/cli/add_partitions.rb

@@ -27,7 +27,7 @@ module PgSlice
       queries = []
 
       if needs_comment
-        queries << "COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_table(table)} is 'column:#{field},period:#{period},cast:#{cast}';"
+        queries << "COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_table(table)} IS 'column:#{field},period:#{period},cast:#{cast}';"
       end
 
       # today = utc date
@@ -96,6 +96,9 @@ CREATE TABLE #{quote_table(partition)}
         partitions.each do |partition|
           day = partition_date(partition, name_format)
 
+          # note: does not support generated columns
+          # could support by listing columns
+          # but this would cause issues with schema changes
           sql = "(NEW.#{quote_ident(field)} >= #{sql_date(day, cast)} AND NEW.#{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}) THEN
               INSERT INTO #{quote_table(partition)} VALUES (NEW.*);"
 

data/lib/pgslice/cli/prep.rb

@@ -3,6 +3,7 @@ module PgSlice
     desc "prep TABLE [COLUMN] [PERIOD]", "Create an intermediate table for partitioning"
     option :partition, type: :boolean, default: true, desc: "Partition the table"
     option :trigger_based, type: :boolean, default: false, desc: "Use trigger-based partitioning"
+    option :test_version, type: :numeric, hide: true
     def prep(table, column=nil, period=nil)
       table = create_table(table)
       intermediate_table = table.intermediate_table
@@ -24,23 +25,23 @@ module PgSlice
       queries = []
 
       # version summary
-      # 1. trigger-based
-      # 2. declarative, with indexes and foreign keys on child tables
-      # 3. declarative, with indexes and foreign keys on parent table
-      version =
-        if options[:trigger_based] || server_version_num < 100000
-          1
-        elsif server_version_num < 110000
-          2
-        else
-          3
-        end
+      # 1. trigger-based (pg9)
+      # 2. declarative, with indexes and foreign keys on child tables (pg10)
+      # 3. declarative, with indexes and foreign keys on parent table (pg11+)
+      version = options[:test_version] || (options[:trigger_based] ? 1 : 3)
 
       declarative = version > 1
 
       if declarative && options[:partition]
+        including = ["DEFAULTS", "CONSTRAINTS", "STORAGE", "COMMENTS", "STATISTICS"]
+        if server_version_num >= 120000
+          including << "GENERATED"
+        end
+        if server_version_num >= 140000
+          including << "COMPRESSION"
+        end
         queries << <<-SQL
-CREATE TABLE #{quote_table(intermediate_table)} (LIKE #{quote_table(table)} INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING STORAGE INCLUDING COMMENTS) PARTITION BY RANGE (#{quote_ident(column)});
+CREATE TABLE #{quote_table(intermediate_table)} (LIKE #{quote_table(table)} #{including.map { |v| "INCLUDING #{v}" }.join(" ")}) PARTITION BY RANGE (#{quote_ident(column)});
         SQL
 
         if version == 3
@@ -57,7 +58,7 @@ CREATE TABLE #{quote_table(intermediate_table)} (LIKE #{quote_table(table)} INCL
         # add comment
         cast = table.column_cast(column)
         queries << <<-SQL
-COMMENT ON TABLE #{quote_table(intermediate_table)} is 'column:#{column},period:#{period},cast:#{cast},version:#{version}';
+COMMENT ON TABLE #{quote_table(intermediate_table)} IS 'column:#{column},period:#{period},cast:#{cast},version:#{version}';
         SQL
       else
         queries << <<-SQL
@@ -87,7 +88,7 @@ CREATE TRIGGER #{quote_ident(trigger_name)}
 
         cast = table.column_cast(column)
         queries << <<-SQL
-COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_table(intermediate_table)} is 'column:#{column},period:#{period},cast:#{cast}';
+COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_table(intermediate_table)} IS 'column:#{column},period:#{period},cast:#{cast}';
         SQL
       end
 

data/lib/pgslice/cli/swap.rb

@@ -20,7 +20,7 @@ module PgSlice
         queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_schema"])}.#{quote_ident(sequence["sequence_name"])} OWNED BY #{quote_table(table)}.#{quote_ident(sequence["related_column"])};"
       end
 
-      queries.unshift("SET LOCAL lock_timeout = '#{options[:lock_timeout]}';") if server_version_num >= 90300
+      queries.unshift("SET LOCAL lock_timeout = #{escape_literal(options[:lock_timeout])};")
 
       run_queries(queries)
     end

data/lib/pgslice/helpers.rb

@@ -35,11 +35,15 @@ module PgSlice
         @schema = Array(params.delete("schema") || "public")[0]
         uri.query = params.any? ? URI.encode_www_form(params) : nil
 
-        ENV["PGCONNECT_TIMEOUT"] ||= "1"
+        ENV["PGCONNECT_TIMEOUT"] ||= "3"
         conn = PG::Connection.new(uri.to_s)
         conn.set_notice_processor do |message|
           say message
         end
+        @server_version_num = conn.exec("SHOW server_version_num")[0]["server_version_num"].to_i
+        if @server_version_num < 110000
+          abort "This version of pgslice requires Postgres 11+"
+        end
         conn
       end
     rescue PG::ConnectionBad => e
@@ -86,7 +90,8 @@ module PgSlice
     end
 
     def server_version_num
-      execute("SHOW server_version_num")[0]["server_version_num"].to_i
+      connection # ensure called first
+      @server_version_num
     end
 
     # helpers
@@ -97,7 +102,7 @@ module PgSlice
       else
         fmt = "%Y-%m-%d"
       end
-      str = "'#{time.strftime(fmt)}'"
+      str = escape_literal(time.strftime(fmt))
       add_cast ? "#{str}::#{cast}" : str
     end
 
@@ -152,6 +157,10 @@ module PgSlice
       PG::Connection.quote_ident(value)
     end
 
+    def escape_literal(value)
+      connection.escape_literal(value)
+    end
+
     def quote_table(table)
       table.quote_table
     end
@@ -176,5 +185,12 @@ module PgSlice
     def make_fk_def(fk_def, table)
       "ALTER TABLE #{quote_table(table)} ADD #{fk_def};"
     end
+
+    def make_stat_def(stat_def, table)
+      m = /ON (.+) FROM/.match(stat_def)
+      # errors on duplicate names, but should be rare
+      stat_name = "#{table}_#{m[1].split(", ").map { |v| v.gsub(/\W/i, "") }.join("_")}_stat"
+      stat_def.sub(/ FROM \S+/, " FROM #{quote_table(table)}").sub(/ STATISTICS .+ ON /, " STATISTICS #{quote_ident(stat_name)} ON ") + ";"
+    end
   end
 end

data/lib/pgslice/table.rb

@@ -16,7 +16,7 @@ module PgSlice
     end
 
     def columns
-      execute("SELECT column_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2", [schema, name]).map{ |r| r["column_name"] }
+      execute("SELECT column_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 AND is_generated = 'NEVER'", [schema, name]).map{ |r| r["column_name"] }
     end
 
     # http://www.dbforums.com/showthread.php?1667561-How-to-list-sequences-and-the-columns-by-SQL
@@ -178,12 +178,16 @@ module PgSlice
       PgSlice::CLI.instance.send(:execute, *args)
     end
 
+    def escape_literal(value)
+      PgSlice::CLI.instance.send(:escape_literal, value)
+    end
+
     def quote_ident(value)
       PG::Connection.quote_ident(value)
     end
 
     def regclass
-      "'#{quote_table}'::regclass"
+      "#{escape_literal(quote_table)}::regclass"
     end
 
     def sql_date(time, cast, add_cast = true)
@@ -192,7 +196,7 @@ module PgSlice
       else
         fmt = "%Y-%m-%d"
       end
-      str = "'#{time.strftime(fmt)}'"
+      str = escape_literal(time.strftime(fmt))
       add_cast ? "#{str}::#{cast}" : str
     end
   end

data/lib/pgslice/version.rb

@@ -1,3 +1,3 @@
 module PgSlice
-  VERSION = "0.4.8"
+  VERSION = "0.6.0"
 end

data/lib/pgslice.rb

@@ -7,16 +7,16 @@ require "cgi"
 require "time"
 
 # modules
-require "pgslice/helpers"
-require "pgslice/table"
-require "pgslice/version"
+require_relative "pgslice/helpers"
+require_relative "pgslice/table"
+require_relative "pgslice/version"
 
 # commands
-require "pgslice/cli"
-require "pgslice/cli/add_partitions"
-require "pgslice/cli/analyze"
-require "pgslice/cli/fill"
-require "pgslice/cli/prep"
-require "pgslice/cli/swap"
-require "pgslice/cli/unprep"
-require "pgslice/cli/unswap"
+require_relative "pgslice/cli"
+require_relative "pgslice/cli/add_partitions"
+require_relative "pgslice/cli/analyze"
+require_relative "pgslice/cli/fill"
+require_relative "pgslice/cli/prep"
+require_relative "pgslice/cli/swap"
+require_relative "pgslice/cli/unprep"
+require_relative "pgslice/cli/unswap"

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: pgslice
 version: !ruby/object:Gem::Version
-  version: 0.4.8
+  version: 0.6.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-02-28 00:00:00.000000000 Z
+date: 2023-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
 - !ruby/object:Gem::Dependency
-  name: pg
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.18.2
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.18.2
+        version: '0'
 description:
 email: andrew@ankane.org
 executables:
@@ -73,14 +73,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.2'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Postgres partitioning as easy as pie