pgslice 0.4.4 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b29363ba9973cca9be997f3bc62c748b27bee61d25da67f3d3af60f9166cec9
-  data.tar.gz: bb6be94d05a1ff1e059f3386ce4ba34a1a4698a25f89e676d8d3117d4d203ac4
+  metadata.gz: 21d2ab39e5d79cc2477aedc2f458f5e42341c76387fcdd97dd7865cfaec777f8
+  data.tar.gz: 9df6923ae6aba4054c6631a3d0c5c34c8de82214db0d0d6644e73002b9224702
 SHA512:
-  metadata.gz: abceaedcfa90cfa78954c3184baa02e270244d89b4be19016eb0e1c570902008745845018120182f4dbe543a42cf8d6d8926a52238245d95976946fcf1a49d6e
-  data.tar.gz: b5adb37599b3651e737f81193f50495a9ae416544cc96acb77f371c26083cae000c27dc740a9992d303815eb3a250881c7ce5c0dd7bf571e7c1b58f4df4dff66
+  metadata.gz: eb51a60758f6179337c74052bac2ba433efc65ec211ccbcbf9175f9986df9e45f3f16560669a29fd6e9b3684377164dea622d5e39e26ac0f442b441e92953f72
+  data.tar.gz: fc16788a99b3f307d99dd1d5230cb53e5746241e68cf3028454301f48cd711332b71d4716ab1210c07ca9a86daa07df342d009b306f24cc6245a9efbcb6fb32b

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.4.5
+
+- Added support for Postgres 11 foreign key improvements
+- Improved versioning
+
 ## 0.4.4
 
 - Added partitioning by `year`

data/README.md

@@ -1,6 +1,6 @@
 # pgslice
 
-Postgres partitioning as easy as pie. Works great for both new and existing tables, with zero downtime and minimal app changes. Archive older data on a rolling basis to keep your database size under control.
+Postgres partitioning as easy as pie. Works great for both new and existing tables, with zero downtime and minimal app changes. No need to install anything on your database server. Archive older data on a rolling basis to keep your database size under control.
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
@@ -92,34 +92,13 @@ pgslice prep visits created_at month
 ```
 
 ```sql
--- Postgres 10
-
-BEGIN;
-
-CREATE TABLE visits_intermediate (LIKE visits INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING STORAGE INCLUDING COMMENTS) PARTITION BY RANGE (created_at);
-
-COMMENT ON TABLE visits_intermediate is 'column:created_at,period:month';
-
-COMMIT;
-
--- Postgres 9.6 and below
-
 BEGIN;
 
-CREATE TABLE visits_intermediate (LIKE visits INCLUDING ALL);
-
-CREATE FUNCTION visits_insert_trigger()
-    RETURNS trigger AS $$
-    BEGIN
-        RAISE EXCEPTION 'Create partitions first.';
-    END;
-    $$ LANGUAGE plpgsql;
+CREATE TABLE "public"."visits_intermediate" (LIKE "public"."visits" INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING STORAGE INCLUDING COMMENTS) PARTITION BY RANGE ("created_at");
 
-CREATE TRIGGER visits_insert_trigger
-    BEFORE INSERT ON visits_intermediate
-    FOR EACH ROW EXECUTE PROCEDURE visits_insert_trigger();
+CREATE INDEX ON "public"."visits_intermediate" USING btree ("created_at");
 
-COMMENT ON TRIGGER visits_insert_trigger ON visits_intermediate is 'column:created_at,period:month';
+COMMENT ON TABLE "public"."visits_intermediate" is 'column:createdAt,period:day,cast:date,version:3';
 
 COMMIT;
 ```
@@ -129,73 +108,19 @@ pgslice add_partitions visits --intermediate --past 1 --future 1
 ```
 
 ```sql
--- Postgres 10
-
-BEGIN;
-
-CREATE TABLE visits_201608 PARTITION OF visits_intermediate FOR VALUES FROM ('2016-08-01') TO ('2016-09-01');
-
-ALTER TABLE visits_201608 ADD PRIMARY KEY (id);
-
-CREATE INDEX ON visits_201608 USING btree (user_id);
-
-CREATE TABLE visits_201609 PARTITION OF visits_intermediate FOR VALUES FROM ('2016-09-01') TO ('2016-10-01');
-
-ALTER TABLE visits_201609 ADD PRIMARY KEY (id);
-
-CREATE INDEX ON visits_201609 USING btree (user_id);
-
-CREATE TABLE visits_201610 PARTITION OF visits_intermediate FOR VALUES FROM ('2016-10-01') TO ('2016-11-01');
-
-ALTER TABLE visits_201610 ADD PRIMARY KEY (id);
-
-CREATE INDEX ON visits_201610 USING btree (user_id);
-
-COMMIT;
-
--- Postgres 9.6 and below
-
 BEGIN;
 
-CREATE TABLE visits_201608
-    (CHECK (created_at >= '2016-08-01'::date AND created_at < '2016-09-01'::date))
-    INHERITS (visits_intermediate);
+CREATE TABLE "public"."visits_201808" PARTITION OF "public"."visits_intermediate" FOR VALUES FROM ('2018-08-01') TO ('2018-09-01');
 
-ALTER TABLE visits_201608 ADD PRIMARY KEY (id);
+ALTER TABLE "public"."visits_201808" ADD PRIMARY KEY ("id");
 
-CREATE INDEX ON visits_201608 USING btree (user_id);
+CREATE TABLE "public"."visits_201809" PARTITION OF "public"."visits_intermediate" FOR VALUES FROM ('2018-09-01') TO ('2018-10-01');
 
-CREATE TABLE visits_201609
-    (CHECK (created_at >= '2016-09-01'::date AND created_at < '2016-10-01'::date))
-    INHERITS (visits_intermediate);
+ALTER TABLE "public"."visits_201809" ADD PRIMARY KEY ("id");
 
-ALTER TABLE visits_201609 ADD PRIMARY KEY (id);
+CREATE TABLE "public"."visits_201810" PARTITION OF "public"."visits_intermediate" FOR VALUES FROM ('2018-10-01') TO ('2018-11-01');
 
-CREATE INDEX ON visits_201609 USING btree (user_id);
-
-CREATE TABLE visits_201610
-    (CHECK (created_at >= '2016-10-01'::date AND created_at < '2016-11-01'::date))
-    INHERITS (visits_intermediate);
-
-ALTER TABLE visits_201610 ADD PRIMARY KEY (id);
-
-CREATE INDEX ON visits_201610 USING btree (user_id);
-
-CREATE OR REPLACE FUNCTION visits_insert_trigger()
-    RETURNS trigger AS $$
-    BEGIN
-        IF (NEW.created_at >= '2016-09-01'::date AND NEW.created_at < '2016-10-01'::date) THEN
-            INSERT INTO visits_201609 VALUES (NEW.*);
-        ELSIF (NEW.created_at >= '2016-10-01'::date AND NEW.created_at < '2016-11-01'::date) THEN
-            INSERT INTO visits_201610 VALUES (NEW.*);
-        ELSIF (NEW.created_at >= '2016-08-01'::date AND NEW.created_at < '2016-09-01'::date) THEN
-            INSERT INTO visits_201608 VALUES (NEW.*);
-        ELSE
-            RAISE EXCEPTION 'Date out of range. Ensure partitions are created.';
-        END IF;
-        RETURN NULL;
-    END;
-    $$ LANGUAGE plpgsql;
+ALTER TABLE "public"."visits_201808" ADD PRIMARY KEY ("id");
 
 COMMIT;
 ```
@@ -206,19 +131,19 @@ pgslice fill visits
 
 ```sql
 /* 1 of 3 */
-INSERT INTO visits_intermediate ("id", "user_id", "ip", "created_at")
-    SELECT "id", "user_id", "ip", "created_at" FROM visits
-    WHERE id > 0 AND id <= 10000 AND created_at >= '2016-08-01'::date AND created_at < '2016-11-01'::date
+INSERT INTO "public"."visits_intermediate" ("id", "user_id", "ip", "created_at")
+    SELECT "id", "user_id", "ip", "created_at" FROM "public"."visits"
+    WHERE "id" > 0 AND "id" <= 10000 AND "created_at" >= '2018-08-01'::date AND "created_at" < '2018-11-01'::date
 
 /* 2 of 3 */
-INSERT INTO visits_intermediate ("id", "user_id", "ip", "created_at")
-    SELECT "id", "user_id", "ip", "created_at" FROM visits
-    WHERE id > 10000 AND id <= 20000 AND created_at >= '2016-08-01'::date AND created_at < '2016-11-01'::date
+INSERT INTO "public"."visits_intermediate" ("id", "user_id", "ip", "created_at")
+    SELECT "id", "user_id", "ip", "created_at" FROM "public"."visits"
+    WHERE "id" > 10000 AND "id" <= 20000 AND "created_at" >= '2018-08-01'::date AND "created_at" < '2018-11-01'::date
 
 /* 3 of 3 */
-INSERT INTO visits_intermediate ("id", "user_id", "ip", "created_at")
-    SELECT "id", "user_id", "ip", "created_at" FROM visits
-    WHERE id > 20000 AND id <= 30000 AND created_at >= '2016-08-01'::date AND created_at < '2016-11-01'::date
+INSERT INTO "public"."visits_intermediate" ("id", "user_id", "ip", "created_at")
+    SELECT "id", "user_id", "ip", "created_at" FROM "public"."visits"
+    WHERE "id" > 20000 AND "id" <= 30000 AND "created_at" >= '2018-08-01'::date AND "created_at" < '2018-11-01'::date
 ```
 
 ```sh
@@ -226,13 +151,13 @@ pgslice analyze visits
 ```
 
 ```sql
-ANALYZE VERBOSE visits_201608;
+ANALYZE VERBOSE "public"."visits_201808";
 
-ANALYZE VERBOSE visits_201609;
+ANALYZE VERBOSE "public"."visits_201809";
 
-ANALYZE VERBOSE visits_201610;
+ANALYZE VERBOSE "public"."visits_201810";
 
-ANALYZE VERBOSE visits_intermediate;
+ANALYZE VERBOSE "public"."visits_intermediate";
 ```
 
 ```sh
@@ -244,11 +169,11 @@ BEGIN;
 
 SET LOCAL lock_timeout = '5s';
 
-ALTER TABLE visits RENAME TO visits_retired;
+ALTER TABLE "public"."visits" RENAME TO "visits_retired";
 
-ALTER TABLE visits_intermediate RENAME TO visits;
+ALTER TABLE "public"."visits_intermediate" RENAME TO "visits";
 
-ALTER SEQUENCE visits_id_seq OWNED BY visits.id;
+ALTER SEQUENCE "visits_id_seq" OWNED BY "public"."visits"."id";
 
 COMMIT;
 ```
@@ -294,14 +219,14 @@ WHERE
 Back up and drop older partitions each day, month, or year.
 
 ```sh
-pg_dump -c -Fc -t <table>_201609 $PGSLICE_URL > <table>_201609.dump
-psql -c "DROP <table>_201609" $PGSLICE_URL
+pg_dump -c -Fc -t <table>_201809 $PGSLICE_URL > <table>_201809.dump
+psql -c "DROP <table>_201809" $PGSLICE_URL
 ```
 
 If you use [Amazon S3](https://aws.amazon.com/s3/) for backups, [s3cmd](https://github.com/s3tools/s3cmd) is a nice tool.
 
 ```sh
-s3cmd put <table>_201609.dump s3://<s3-bucket>/<table>_201609.dump
+s3cmd put <table>_201809.dump s3://<s3-bucket>/<table>_201809.dump
 ```
 
 ## Additional Commands
@@ -322,16 +247,9 @@ pgslice unswap <table>
 
 This set up allows you to read and write with the original table name with no knowledge it’s partitioned. However, there are a few things to be aware of.
 
-### Writes
-
-Before Postgres 10, if you use `INSERT` statements with a `RETURNING` clause (as frameworks like Rails do), you’ll no longer receive the id of the newly inserted record(s) back. If you need this, you can either:
-
-1. Insert directly into the partition
-2. Get value before the insert with `SELECT nextval('sequence_name')` (for multiple rows, append `FROM generate_series(1, n)`)
-
 ### Reads
 
-When possible, queries should include the column you partition on to limit the number of partitions the database needs to check.  For instance, if you partition on `created_at`, try to include it in queries:
+When possible, queries should include the column you partition on to limit the number of partitions the database needs to check. For instance, if you partition on `created_at`, try to include it in queries:
 
 ```sql
 SELECT * FROM
@@ -339,7 +257,7 @@ SELECT * FROM
 WHERE
     user_id = 123 AND
     -- for performance
-    created_at >= '2016-09-01' AND created_at < '2016-09-02'
+    created_at >= '2018-09-01' AND created_at < '2018-09-02'
 ```
 
 For this to be effective, ensure `constraint_exclusion` is set to `partition` (default value) or `on`.
@@ -348,6 +266,13 @@ For this to be effective, ensure `constraint_exclusion` is set to `partition` (d
 SHOW constraint_exclusion;
 ```
 
+### Writes
+
+Before Postgres 10, if you use `INSERT` statements with a `RETURNING` clause (as frameworks like Rails do), you’ll no longer receive the id of the newly inserted record(s) back. If you need this, you can either:
+
+1. Insert directly into the partition
+2. Get value before the insert with `SELECT nextval('sequence_name')` (for multiple rows, append `FROM generate_series(1, n)`)
+
 ## One Off Tasks
 
 You can also use pgslice to reduce the size of a table without partitioning by creating a new table, filling it with a subset of records, and swapping it in.
@@ -362,8 +287,8 @@ pgslice swap <table>
 
 Once a table is partitioned, here’s how to change the schema:
 
-- To add, remove, or modify a column, make the update on the master table only
-- To add or remove an index, make the update on the master table and all partitions (for Postgres 11, make the update on the master table only)
+- To add, remove, or modify a column, make the update on the master table only.
+- To add or remove an index or foreign key, make the update on the master table. For Postgres < 11, make the update on all partitions as well.
 
 ## Declarative Partitioning
 
@@ -371,17 +296,7 @@ Postgres 10 introduces [declarative partitioning](https://www.postgresql.org/doc
 
 ## Data Protection
 
-When connecting to a remote database, make sure your connection is secure.
-
-If you do not use a VPN, you must use `sslmode=verify-full` with a root certificate to [protect against MITM attacks](https://www.postgresql.org/docs/current/static/libpq-ssl.html). If you don’t do this, your database credentials can be compromised. This cannot be understated!
-
-Surprisingly and unfortunately, there’s [not a secure way](https://thusoy.com/2016/mitming-postgres) to connect to Heroku Postgres with any client.
-
-For Amazon RDS, download the [root certificate](https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem) and append to your database url:
-
-```
-?sslmode=verify-full&sslrootcert=rds-combined-ca-bundle.pem
-```
+Always make sure your connection is secure when connecting to your database over a network you don’t fully trust. Your best option is to connect over SSH or a VPN. Another option is to use `sslmode=verify-full`. If you don’t do this, your database credentials can be compromised.
 
 ## Upgrading
 

data/exe/pgslice

@@ -2,7 +2,7 @@
 
 require "pgslice"
 begin
-  PgSlice::Client.start
+  PgSlice::CLI.start
 rescue Interrupt
   abort
 end

data/lib/pgslice.rb

@@ -1,10 +1,20 @@
 # dependencies
 require "cgi"
-require "thor"
 require "pg"
+require "thor"
+require "time"
 
 # modules
-require "pgslice/client"
-require "pgslice/generic_table"
+require "pgslice/helpers"
 require "pgslice/table"
 require "pgslice/version"
+
+# commands
+require "pgslice/cli"
+require "pgslice/cli/add_partitions"
+require "pgslice/cli/analyze"
+require "pgslice/cli/fill"
+require "pgslice/cli/prep"
+require "pgslice/cli/swap"
+require "pgslice/cli/unprep"
+require "pgslice/cli/unswap"

data/lib/pgslice/cli.rb

@@ -0,0 +1,32 @@
+module PgSlice
+  class CLI < Thor
+    class << self
+      attr_accessor :instance
+    end
+
+    include Helpers
+
+    check_unknown_options!
+
+    class_option :url, desc: "Database URL"
+    class_option :dry_run, type: :boolean, default: false, desc: "Print statements without executing"
+
+    map %w[--version -v] => :version
+
+    def self.exit_on_failure?
+      true
+    end
+
+    def initialize(*args)
+      PgSlice::CLI.instance = self
+      $stdout.sync = true
+      $stderr.sync = true
+      super
+    end
+
+    desc "version", "Show version"
+    def version
+      log("pgslice #{PgSlice::VERSION}")
+    end
+  end
+end

data/lib/pgslice/cli/add_partitions.rb

@@ -0,0 +1,130 @@
+module PgSlice
+  class CLI
+    desc "add_partitions TABLE", "Add partitions"
+    option :intermediate, type: :boolean, default: false, desc: "Add to intermediate table"
+    option :past, type: :numeric, default: 0, desc: "Number of past partitions to add"
+    option :future, type: :numeric, default: 0, desc: "Number of future partitions to add"
+    def add_partitions(table)
+      original_table = create_table(table)
+      table = options[:intermediate] ? original_table.intermediate_table : original_table
+      trigger_name = original_table.trigger_name
+
+      assert_table(table)
+
+      future = options[:future]
+      past = options[:past]
+      range = (-1 * past)..future
+
+      period, field, cast, needs_comment, declarative, version = table.fetch_settings(original_table.trigger_name)
+      unless period
+        message = "No settings found: #{table}"
+        message = "#{message}\nDid you mean to use --intermediate?" unless options[:intermediate]
+        abort message
+      end
+
+      queries = []
+
+      if needs_comment
+        queries << "COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_table(table)} is 'column:#{field},period:#{period},cast:#{cast}';"
+      end
+
+      # today = utc date
+      today = round_date(Time.now.utc.to_date, period)
+
+      schema_table =
+        if !declarative
+          table
+        elsif options[:intermediate]
+          original_table
+        else
+          table.partitions.last
+        end
+
+      # indexes automatically propagate in Postgres 11+
+      if version < 3
+        index_defs = schema_table.index_defs
+        fk_defs = schema_table.foreign_keys
+      else
+        index_defs = []
+        fk_defs = []
+      end
+
+      primary_key = schema_table.primary_key
+
+      added_partitions = []
+      range.each do |n|
+        day = advance_date(today, period, n)
+
+        partition = Table.new(original_table.schema, "#{original_table.name}_#{day.strftime(name_format(period))}")
+        next if partition.exists?
+        added_partitions << partition
+
+        if declarative
+          queries << <<-SQL
+CREATE TABLE #{quote_table(partition)} PARTITION OF #{quote_table(table)} FOR VALUES FROM (#{sql_date(day, cast, false)}) TO (#{sql_date(advance_date(day, period, 1), cast, false)});
+          SQL
+        else
+          queries << <<-SQL
+CREATE TABLE #{quote_table(partition)}
+    (CHECK (#{quote_ident(field)} >= #{sql_date(day, cast)} AND #{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}))
+    INHERITS (#{quote_table(table)});
+          SQL
+        end
+
+        queries << "ALTER TABLE #{quote_table(partition)} ADD PRIMARY KEY (#{primary_key.map { |k| quote_ident(k) }.join(", ")});" if primary_key.any?
+
+        index_defs.each do |index_def|
+          queries << make_index_def(index_def, partition)
+        end
+
+        fk_defs.each do |fk_def|
+          queries << make_fk_def(fk_def, partition)
+        end
+      end
+
+      unless declarative
+        # update trigger based on existing partitions
+        current_defs = []
+        future_defs = []
+        past_defs = []
+        name_format = self.name_format(period)
+        partitions = (table.partitions + added_partitions).uniq(&:name).sort_by(&:name)
+
+        partitions.each do |partition|
+          day = partition_date(partition, name_format)
+
+          sql = "(NEW.#{quote_ident(field)} >= #{sql_date(day, cast)} AND NEW.#{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}) THEN
+              INSERT INTO #{quote_table(partition)} VALUES (NEW.*);"
+
+          if day.to_date < today
+            past_defs << sql
+          elsif advance_date(day, period, 1) < today
+            current_defs << sql
+          else
+            future_defs << sql
+          end
+        end
+
+        # order by current period, future periods asc, past periods desc
+        trigger_defs = current_defs + future_defs + past_defs.reverse
+
+        if trigger_defs.any?
+          queries << <<-SQL
+CREATE OR REPLACE FUNCTION #{quote_ident(trigger_name)}()
+    RETURNS trigger AS $$
+    BEGIN
+        IF #{trigger_defs.join("\n        ELSIF ")}
+        ELSE
+            RAISE EXCEPTION 'Date out of range. Ensure partitions are created.';
+        END IF;
+        RETURN NULL;
+    END;
+    $$ LANGUAGE plpgsql;
+          SQL
+        end
+      end
+
+      run_queries(queries) if queries.any?
+    end
+  end
+end