pgslice 0.3.6 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 07d87eec1238827086898708a5b4e026746a0c11
-  data.tar.gz: 76024edd24065ec3e24b74d72563583e01cecc6e
+  metadata.gz: 9db8d56e7e5ea8f2b5769bea524b2db90731c8a3
+  data.tar.gz: 4d05fabf0ded181cc1ab972988d411fb22518168
 SHA512:
-  metadata.gz: 9c6042dc4fe0d093c3b38c05d91b445ad1882241f5c3b814f4fe7124ae3609b591ead4d70de39ebf496164e72221ffb409703173689a0daa6f37f884e73dce45
-  data.tar.gz: b79e389ab3529ad08eadb2e18b6c374d40436007e4fe788e2898888e5caf5592b3839326fee9c4d8d658bbfd284c0c953683171d8789de826b86fa79b7d3354c
+  metadata.gz: c698c0b5176fb902460595f5702d37ea612cc60ddc1438a164810023e540e8daf9a9cff0b505ecbc770e9e4f25c568be0daeb91f86f975fa961b42a1dec9ed10
+  data.tar.gz: 761b3365b4bb1fe85acfd737ad18fa5e1c7938162dabf20c68d53115dfbbfa429d075d885b5bb6c7b6a3ba71fb38cb84187456c12fb0134ca06a45e08a7c3493

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+rvm: 2.4.1
+cache: bundler
+sudo: false
+script: bundle exec rake test
+addons:
+  postgresql: "9.4"
+before_script:
+  - psql -c 'create database pgslice_test;' -U postgres
+notifications:
+  email:
+    on_success: never
+    on_failure: change

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.4.0
+
+- Added support for declarative partitioning
+- Added support for foreign keys
+
 ## 0.3.6
 
 - Fixed drop trigger on `unprep` for non-lowercase tables

data/README.md

@@ -4,6 +4,8 @@ Postgres partitioning as easy as pie. Works great for both new and existing tabl
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
+[![Build Status](https://travis-ci.org/ankane/pgslice.svg?branch=master)](https://travis-ci.org/ankane/pgslice)
+
 ## Install
 
 pgslice is a command line tool. To install, run:
@@ -280,7 +282,7 @@ This set up allows you to read and write with the original table name with no kn
 
 ### Writes
 
-If you use `INSERT` statements with a `RETURNING` clause (as frameworks like Rails do), you’ll no longer receive the id of the newly inserted record(s) back. If you need this, you can either:
+Before Postgres 10, if you use `INSERT` statements with a `RETURNING` clause (as frameworks like Rails do), you’ll no longer receive the id of the newly inserted record(s) back. If you need this, you can either:
 
 1. Insert directly into the partition
 2. Get value before the insert with `SELECT nextval('sequence_name')` (for multiple rows, append `FROM generate_series(1, n)`)
@@ -314,6 +316,17 @@ pgslice fill <table> --where "id > 1000" # use any conditions
 pgslice swap <table>
 ```
 
+## Schema Updates
+
+Once a table is partitioned, here’s how to change the schema:
+
+- To add, remove, or modify a column, make the update on the master table only
+- To add or remove an index, make the update on the master table and all partitions
+
+## Declarative Partitioning
+
+Postgres 10 introduces [declarative partitioning](https://www.postgresql.org/docs/10/static/ddl-partitioning.html#ddl-partitioning-declarative). A major benefit is `INSERT` statements with a `RETURNING` clause work as expected. If you prefer to use trigger-based partitioning instead (not recommended), pass the `--trigger-based` option to the `prep` command.
+
 ## Upgrading
 
 Run:
@@ -364,3 +377,13 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 - Fix bugs and [submit pull requests](https://github.com/ankane/pgslice/pulls)
 - Write, clarify, or fix documentation
 - Suggest or add new features
+
+To run tests, do:
+
+```sh
+git clone https://github.com/ankane/pgslice.git
+cd pgslice
+bundle install
+createdb pgslice_test
+bundle exec rake
+```

data/lib/pgslice.rb

@@ -59,6 +59,7 @@ module PgSlice
 
       if options[:no_partition]
         abort "Usage: pgslice prep <table> --no-partition" if arguments.length != 1
+        abort "Can't use --trigger-based and --no-partition" if options[:trigger_based]
       else
         abort "Usage: pgslice prep <table> <column> <period>" if arguments.length != 3
       end
@@ -72,11 +73,29 @@ module PgSlice
 
       queries = []
 
-      queries << <<-SQL
+      declarative = server_version_num >= 100000 && !options[:trigger_based]
+
+      if declarative && !options[:no_partition]
+        queries << <<-SQL
+CREATE TABLE #{quote_ident(intermediate_table)} (LIKE #{quote_ident(table)} INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING STORAGE INCLUDING COMMENTS) PARTITION BY RANGE (#{quote_ident(column)});
+        SQL
+
+        # add comment
+        cast = column_cast(table, column)
+        queries << <<-SQL
+COMMENT ON TABLE #{quote_ident(intermediate_table)} is 'column:#{column},period:#{period},cast:#{cast}';
+        SQL
+      else
+        queries << <<-SQL
 CREATE TABLE #{quote_ident(intermediate_table)} (LIKE #{quote_ident(table)} INCLUDING ALL);
-      SQL
+        SQL
 
-      unless options[:no_partition]
+        foreign_keys(table).each do |fk_def|
+          queries << "ALTER TABLE #{quote_ident(intermediate_table)} ADD #{fk_def};"
+        end
+      end
+
+      if !options[:no_partition] && !declarative
         sql_format = SQL_FORMAT[period.to_sym]
         queries << <<-SQL
 CREATE FUNCTION #{quote_ident(trigger_name)}()
@@ -91,12 +110,12 @@ CREATE FUNCTION #{quote_ident(trigger_name)}()
 CREATE TRIGGER #{quote_ident(trigger_name)}
     BEFORE INSERT ON #{quote_ident(intermediate_table)}
     FOR EACH ROW EXECUTE PROCEDURE #{quote_ident(trigger_name)}();
-      SQL
+        SQL
 
         cast = column_cast(table, column)
         queries << <<-SQL
 COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_ident(intermediate_table)} is 'column:#{column},period:#{period},cast:#{cast}';
-SQL
+        SQL
       end
 
       run_queries(queries)
@@ -129,23 +148,34 @@ SQL
       past = options[:past]
       range = (-1 * past)..future
 
-      # ensure table has trigger
-      abort "No trigger on table: #{table}\nDid you mean to use --intermediate?" unless has_trigger?(trigger_name, table)
-
-      index_defs = execute("SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indrelid = #{regclass(schema, original_table)} AND indisprimary = 'f'").map { |r| r["pg_get_indexdef"] }
-      primary_key = self.primary_key(table)
+      period, field, cast, needs_comment, declarative = settings_from_trigger(original_table, table)
+      unless period
+        message = "No settings found: #{table}"
+        message = "#{message}\nDid you mean to use --intermediate?" unless options[:intermediate]
+        abort message
+      end
 
       queries = []
 
-      period, field, cast, needs_comment = settings_from_trigger(original_table, table)
-      abort "Could not read settings" unless period
-
       if needs_comment
         queries << "COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_ident(table)} is 'column:#{field},period:#{period},cast:#{cast}';"
       end
 
       # today = utc date
       today = round_date(DateTime.now.new_offset(0).to_date, period)
+
+      schema_table =
+        if !declarative
+          table
+        elsif options[:intermediate]
+          original_table
+        else
+          "#{original_table}_#{today.strftime(name_format(period))}"
+        end
+      index_defs = execute("SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indrelid = #{regclass(schema, schema_table)} AND indisprimary = 'f'").map { |r| r["pg_get_indexdef"] }
+      fk_defs = foreign_keys(schema_table)
+      primary_key = self.primary_key(schema_table)
+
       added_partitions = []
       range.each do |n|
         day = advance_date(today, period, n)
@@ -154,48 +184,59 @@ SQL
         next if table_exists?(partition_name)
         added_partitions << partition_name
 
-        queries << <<-SQL
+        if declarative
+          queries << <<-SQL
+CREATE TABLE #{quote_ident(partition_name)} PARTITION OF #{quote_ident(table)} FOR VALUES FROM (#{sql_date(day, cast, false)}) TO (#{sql_date(advance_date(day, period, 1), cast, false)});
+          SQL
+        else
+          queries << <<-SQL
 CREATE TABLE #{quote_ident(partition_name)}
     (CHECK (#{quote_ident(field)} >= #{sql_date(day, cast)} AND #{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}))
     INHERITS (#{quote_ident(table)});
-        SQL
+          SQL
+        end
 
         queries << "ALTER TABLE #{quote_ident(partition_name)} ADD PRIMARY KEY (#{quote_ident(primary_key)});" if primary_key
 
         index_defs.each do |index_def|
           queries << index_def.sub(/ ON \S+ USING /, " ON #{quote_ident(partition_name)} USING ").sub(/ INDEX .+ ON /, " INDEX ON ") + ";"
         end
-      end
 
-      # update trigger based on existing partitions
-      current_defs = []
-      future_defs = []
-      past_defs = []
-      name_format = self.name_format(period)
-      existing_tables = existing_partitions(original_table)
-      existing_tables = (existing_tables + added_partitions).uniq.sort
-
-      existing_tables.each do |table|
-        day = DateTime.strptime(table.split("_").last, name_format)
-        partition_name = "#{original_table}_#{day.strftime(name_format(period))}"
-
-        sql = "(NEW.#{quote_ident(field)} >= #{sql_date(day, cast)} AND NEW.#{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}) THEN
-            INSERT INTO #{quote_ident(partition_name)} VALUES (NEW.*);"
-
-        if day.to_date < today
-          past_defs << sql
-        elsif advance_date(day, period, 1) < today
-          current_defs << sql
-        else
-          future_defs << sql
+        fk_defs.each do |fk_def|
+          queries << "ALTER TABLE #{quote_ident(partition_name)} ADD #{fk_def};"
         end
       end
 
-      # order by current period, future periods asc, past periods desc
-      trigger_defs = current_defs + future_defs + past_defs.reverse
+      unless declarative
+        # update trigger based on existing partitions
+        current_defs = []
+        future_defs = []
+        past_defs = []
+        name_format = self.name_format(period)
+        existing_tables = existing_partitions(original_table)
+        existing_tables = (existing_tables + added_partitions).uniq.sort
+
+        existing_tables.each do |table|
+          day = DateTime.strptime(table.split("_").last, name_format)
+          partition_name = "#{original_table}_#{day.strftime(name_format(period))}"
+
+          sql = "(NEW.#{quote_ident(field)} >= #{sql_date(day, cast)} AND NEW.#{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}) THEN
+              INSERT INTO #{quote_ident(partition_name)} VALUES (NEW.*);"
+
+          if day.to_date < today
+            past_defs << sql
+          elsif advance_date(day, period, 1) < today
+            current_defs << sql
+          else
+            future_defs << sql
+          end
+        end
+
+        # order by current period, future periods asc, past periods desc
+        trigger_defs = current_defs + future_defs + past_defs.reverse
 
-      if trigger_defs.any?
-        queries << <<-SQL
+        if trigger_defs.any?
+          queries << <<-SQL
 CREATE OR REPLACE FUNCTION #{quote_ident(trigger_name)}()
     RETURNS trigger AS $$
     BEGIN
@@ -206,7 +247,8 @@ CREATE OR REPLACE FUNCTION #{quote_ident(trigger_name)}()
         RETURN NULL;
     END;
     $$ LANGUAGE plpgsql;
-        SQL
+          SQL
+        end
       end
 
       run_queries(queries) if queries.any?
@@ -231,7 +273,7 @@ CREATE OR REPLACE FUNCTION #{quote_ident(trigger_name)}()
       abort "Table not found: #{source_table}" unless table_exists?(source_table)
       abort "Table not found: #{dest_table}" unless table_exists?(dest_table)
 
-      period, field, cast, needs_comment = settings_from_trigger(table, dest_table)
+      period, field, cast, needs_comment, declarative = settings_from_trigger(table, dest_table)
 
       if period
         name_format = self.name_format(period)
@@ -243,7 +285,8 @@ CREATE OR REPLACE FUNCTION #{quote_ident(trigger_name)}()
         end
       end
 
-      primary_key = self.primary_key(table)
+      schema_table = period && declarative ? existing_tables.last : table
+      primary_key = self.primary_key(schema_table)
       abort "No primary key" unless primary_key
       max_source_id = max_id(source_table, primary_key)
 
@@ -368,6 +411,7 @@ INSERT INTO #{quote_ident(dest_table)} (#{fields})
         o.integer "--batch-size", default: 10000
         o.boolean "--dry-run", default: false
         o.boolean "--no-partition", default: false
+        o.boolean "--trigger-based", default: false
         o.integer "--start"
         o.string "--url"
         o.string "--source-table"
@@ -524,6 +568,10 @@ INSERT INTO #{quote_ident(dest_table)} (#{fields})
       !fetch_trigger(trigger_name, table).nil?
     end
 
+    def fetch_comment(table)
+      execute("SELECT obj_description(#{regclass(schema, table)}) AS comment")[0]
+    end
+
     # http://www.dbforums.com/showthread.php?1667561-How-to-list-sequences-and-the-columns-by-SQL
     def sequences(table)
       query = <<-SQL
@@ -561,13 +609,14 @@ INSERT INTO #{quote_ident(dest_table)} (#{fields})
       data_type == "timestamp with time zone" ? "timestamptz" : "date"
     end
 
-    def sql_date(time, cast)
+    def sql_date(time, cast, add_cast = true)
       if cast == "timestamptz"
         fmt = "%Y-%m-%d %H:%M:%S UTC"
       else
         fmt = "%Y-%m-%d"
       end
-      "'#{time.strftime(fmt)}'::#{cast}"
+      str = "'#{time.strftime(fmt)}'"
+      add_cast ? "#{str}::#{cast}" : str
     end
 
     def name_format(period)
@@ -615,7 +664,8 @@ INSERT INTO #{quote_ident(dest_table)} (#{fields})
       trigger_name = self.trigger_name(original_table)
 
       needs_comment = false
-      comment = fetch_trigger(trigger_name, table)
+      trigger_comment = fetch_trigger(trigger_name, table)
+      comment = trigger_comment || fetch_comment(table)
       if comment
         field, period, cast = comment["comment"].split(",").map { |v| v.split(":").last } rescue [nil, nil, nil]
       end
@@ -623,10 +673,10 @@ INSERT INTO #{quote_ident(dest_table)} (#{fields})
       unless period
         needs_comment = true
         function_def = execute("select pg_get_functiondef(oid) from pg_proc where proname = $1", [trigger_name])[0]
-        return [nil, nil] unless function_def
+        return [] unless function_def
         function_def = function_def["pg_get_functiondef"]
         sql_format = SQL_FORMAT.find { |_, f| function_def.include?("'#{f}'") }
-        return [nil, nil] unless sql_format
+        return [] unless sql_format
         period = sql_format[0]
         field = /to_char\(NEW\.(\w+),/.match(function_def)[1]
       end
@@ -638,7 +688,15 @@ INSERT INTO #{quote_ident(dest_table)} (#{fields})
         needs_comment = true
       end
 
-      [period, field, cast, needs_comment]
+      [period, field, cast, needs_comment, !trigger_comment]
+    end
+
+    def foreign_keys(table)
+      execute("SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid = #{regclass(schema, table)} AND contype ='f'").map { |r| r["pg_get_constraintdef"] }
+    end
+
+    def server_version_num
+      execute("SHOW server_version_num").first["server_version_num"].to_i
     end
   end
 end

data/lib/pgslice/version.rb

@@ -1,3 +1,3 @@
 module PgSlice
-  VERSION = "0.3.6"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgslice
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.4.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-10 00:00:00.000000000 Z
+date: 2017-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -89,6 +89,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
 - CHANGELOG.md
 - Dockerfile
 - Gemfile
@@ -117,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Postgres partitioning as easy as pie