pgslice 0.4.5 → 0.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 21d2ab39e5d79cc2477aedc2f458f5e42341c76387fcdd97dd7865cfaec777f8
-  data.tar.gz: 9df6923ae6aba4054c6631a3d0c5c34c8de82214db0d0d6644e73002b9224702
+  metadata.gz: 4d3a279841af6ee62759c9570149cda73460c674c383aef78d7b99fa5c9840ec
+  data.tar.gz: 48462eb02783c097d0553071880e75650263d662cf755e8fa977b1d4e25ccc0e
 SHA512:
-  metadata.gz: eb51a60758f6179337c74052bac2ba433efc65ec211ccbcbf9175f9986df9e45f3f16560669a29fd6e9b3684377164dea622d5e39e26ac0f442b441e92953f72
-  data.tar.gz: fc16788a99b3f307d99dd1d5230cb53e5746241e68cf3028454301f48cd711332b71d4716ab1210c07ca9a86daa07df342d009b306f24cc6245a9efbcb6fb32b
+  metadata.gz: 9c95fe04d9c90f524c9cd778f5a5e05013ff4655982cd4ed1420a024c5c17a97d3de679c86e1d66800a9ee3ab7e29eec8abec8676041d7e07d7ebce96052315f
+  data.tar.gz: 5d43e4d5dc94f29538abd25bd2e3676dc47e1b5cdce646e345539d7543f4b21fb2d9d33ecb08c69df81283150643ad423817179719f712d963b0aba791c10d89

data/CHANGELOG.md

@@ -1,122 +1,137 @@
-## 0.4.5
+## 0.4.8 (2022-02-28)
+
+- Fixed error with pg 1.3
+- Reduced size of Docker image
+
+## 0.4.7 (2020-08-14)
+
+- Added `--tablespace` option to `add_partitions`
+- Fixed sequence query if sequence in different schema than table
+
+## 0.4.6 (2020-05-29)
+
+- Ensure correct order with multi-column primary keys
+- Ensure fill always uses correct date range (bug introduced in 0.4.5)
+
+## 0.4.5 (2018-10-18)
 
 - Added support for Postgres 11 foreign key improvements
 - Improved versioning
 
-## 0.4.4
+## 0.4.4 (2018-08-18)
 
 - Added partitioning by `year`
 - Fixed `--source-table` and `--dest-table` options
 - Added descriptions to options
 
-## 0.4.3
+## 0.4.3 (2018-08-16)
 
 - Fixed sequence ownership
 - Improved help
 
-## 0.4.2
+## 0.4.2 (2018-07-23)
 
 - Added support for Postgres 11 index improvements
 - Added support for all connection options
 
-## 0.4.1
+## 0.4.1 (2018-04-30)
 
 - Better support for schemas
 - Use latest partition for schema
 - Added support for composite primary keys
 
-## 0.4.0
+## 0.4.0 (2017-10-07)
 
 - Added support for declarative partitioning
 - Added support for foreign keys
 
-## 0.3.6
+## 0.3.6 (2017-07-10)
 
 - Fixed drop trigger on `unprep` for non-lowercase tables
 - Fixed index creation for non-lowercase tables
 
-## 0.3.5
+## 0.3.5 (2017-07-06)
 
 - Added support for non-lowercase tables and columns
 
-## 0.3.4
+## 0.3.4 (2017-07-06)
 
 - Added `analyze` method
 - Fixed `fill` with `--dry-run` option
 - Better error message for tables without primary key
 
-## 0.3.3
+## 0.3.3 (2017-03-22)
 
 - Fixed error when creating partitions
 
-## 0.3.2
+## 0.3.2 (2016-12-15)
 
 - Exit with error code on interrupt
 - Fixed `--start` option with `--swapped`
 
-## 0.3.1
+## 0.3.1 (2016-12-13)
 
 - Fixed exception with `--no-partition` option
 - Use proper cast type in `fill` method for legacy `timestamptz` columns
 
-## 0.3.0
+## 0.3.0 (2016-12-12)
 
 - Better query performance for `timestamptz` columns
 - Added support for schemas other than `public`
 
-## 0.2.3
+## 0.2.3 (2016-10-10)
 
 - Added `--dest-table` option to `fill`
 - Fixed errors with `fill` when no partitions created
 
-## 0.2.2
+## 0.2.2 (2016-10-06)
 
 - Set `lock_timeout` on `swap` to prevent bad things from happening
 - Friendlier error messages
 
-## 0.2.1
+## 0.2.1 (2016-09-28)
 
 - Added `--where` option to `fill`
 - Fixed partition detection with `fill`
 - Fixed error for columns named `user` with `fill`
 
-## 0.2.0
+## 0.2.0 (2016-09-22)
 
 - Switched to new trigger, which is about 20% faster
 
-## 0.1.7
+## 0.1.7 (2016-09-14)
 
 - Added `--source-table` option to `fill`
 
-## 0.1.6
+## 0.1.6 (2016-08-04)
 
 - Added `--no-partition` option to `prep`
 - Added `--url` option
 
-## 0.1.5
+## 0.1.5 (2016-04-26)
 
 - Removed `activesupport` dependency for speed
 - Fixed `fill` for months
 
-## 0.1.4
+## 0.1.4 (2016-04-24)
 
 - Added sequence ownership
 - Default to 0 for `--past` and `--future` options
 - Better `fill` with `--swapped`
 
-## 0.1.3
+## 0.1.3 (2016-04-24)
 
 - Fixed table inheritance
 
-## 0.1.2
+## 0.1.2 (2016-04-24)
 
 - Added `--dry-run` option
 - Print sql to stdout instead of stderr
 
-## 0.1.1
+## 0.1.1 (2016-04-24)
 
 - Added sql commands to output
 
-## 0.1.0
+## 0.1.0 (2016-04-24)
 
 - First release

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016-2018 Andrew Kane
+Copyright (c) 2016-2022 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -4,7 +4,7 @@ Postgres partitioning as easy as pie. Works great for both new and existing tabl
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
-[![Build Status](https://travis-ci.org/ankane/pgslice.svg?branch=master)](https://travis-ci.org/ankane/pgslice)
+[![Build Status](https://github.com/ankane/pgslice/workflows/build/badge.svg?branch=master)](https://github.com/ankane/pgslice/actions)
 
 ## Install
 
@@ -14,7 +14,7 @@ pgslice is a command line tool. To install, run:
 gem install pgslice
 ```
 
-This will give you the `pgslice` command.
+This will give you the `pgslice` command. You can also install it with [Homebrew](#homebrew). If installation fails, you may need to install [dependencies](#dependencies).
 
 ## Steps
 
@@ -34,16 +34,14 @@ This will give you the `pgslice` command.
 
   Period can be `day`, `month`, or `year`.
 
-  This creates a table named `<table>_intermediate` with the appropriate trigger for partitioning.
+  This creates a partitioned table named `<table>_intermediate`.
 
-4. Add partitions
+4. Add partitions to the intermediate table
 
   ```sh
   pgslice add_partitions <table> --intermediate --past 3 --future 3
   ```
 
-  This creates child tables that inherit from the intermediate table.
-
   Use the `--past` and `--future` options to control the number of partitions.
 
 5. *Optional, for tables with data* - Fill the partitions in batches with data from the original table
@@ -80,7 +78,7 @@ This will give you the `pgslice` command.
 
   ```sql
   pg_dump -c -Fc -t <table>_retired $PGSLICE_URL > <table>_retired.dump
-  psql -c "DROP <table>_retired" $PGSLICE_URL
+  psql -c "DROP TABLE <table>_retired" $PGSLICE_URL
   ```
 
 ## Sample Output
@@ -173,7 +171,7 @@ ALTER TABLE "public"."visits" RENAME TO "visits_retired";
 
 ALTER TABLE "public"."visits_intermediate" RENAME TO "visits";
 
-ALTER SEQUENCE "visits_id_seq" OWNED BY "public"."visits"."id";
+ALTER SEQUENCE "public"."visits_id_seq" OWNED BY "public"."visits"."id";
 
 COMMIT;
 ```
@@ -220,7 +218,7 @@ Back up and drop older partitions each day, month, or year.
 
 ```sh
 pg_dump -c -Fc -t <table>_201809 $PGSLICE_URL > <table>_201809.dump
-psql -c "DROP <table>_201809" $PGSLICE_URL
+psql -c "DROP TABLE <table>_201809" $PGSLICE_URL
 ```
 
 If you use [Amazon S3](https://aws.amazon.com/s3/) for backups, [s3cmd](https://github.com/s3tools/s3cmd) is a nice tool.
@@ -229,6 +227,15 @@ If you use [Amazon S3](https://aws.amazon.com/s3/) for backups, [s3cmd](https://
 s3cmd put <table>_201809.dump s3://<s3-bucket>/<table>_201809.dump
 ```
 
+## Schema Updates
+
+Once a table is partitioned, make schema updates on the master table only (not partitions). This includes adding, removing, and modifying columns, as well as adding and removing indexes and foreign keys.
+
+A few exceptions are:
+
+- For Postgres 10, make index and foreign key updates on partitions only
+- For Postgres < 10, make index and foreign key updates on the master table and all partitions
+
 ## Additional Commands
 
 To undo prep (which will delete partitions), use:
@@ -243,6 +250,14 @@ To undo swap, use:
 pgslice unswap <table>
 ```
 
+## Additional Options
+
+Set the tablespace when adding partitions
+
+```sh
+pgslice add_partitions <table> --tablespace fastspace
+```
+
 ## App Considerations
 
 This set up allows you to read and write with the original table name with no knowledge it’s partitioned. However, there are a few things to be aware of.
@@ -273,6 +288,32 @@ Before Postgres 10, if you use `INSERT` statements with a `RETURNING` clause (as
 1. Insert directly into the partition
 2. Get value before the insert with `SELECT nextval('sequence_name')` (for multiple rows, append `FROM generate_series(1, n)`)
 
+## Frameworks
+
+### Rails
+
+For Postgres 10+, specify the primary key for partitioned models to ensure it’s returned.
+
+```ruby
+class Visit < ApplicationRecord
+  self.primary_key = "id"
+end
+```
+
+Before Postgres 10, preload the value.
+
+```ruby
+class Visit < ApplicationRecord
+  before_create do
+    self.id ||= self.class.connection.select_all("SELECT nextval('#{self.class.sequence_name}')").first["nextval"]
+  end
+end
+```
+
+### Other Frameworks
+
+Please submit a PR if additional configuration is needed.
+
 ## One Off Tasks
 
 You can also use pgslice to reduce the size of a table without partitioning by creating a new table, filling it with a subset of records, and swapping it in.
@@ -283,12 +324,9 @@ pgslice fill <table> --where "id > 1000" # use any conditions
 pgslice swap <table>
 ```
 
-## Schema Updates
+## Triggers
 
-Once a table is partitioned, here’s how to change the schema:
-
-- To add, remove, or modify a column, make the update on the master table only.
-- To add or remove an index or foreign key, make the update on the master table. For Postgres < 11, make the update on all partitions as well.
+Triggers aren’t copied from the original table. You can set up triggers on the intermediate table if needed. Note that Postgres doesn’t support `BEFORE / FOR EACH ROW` triggers on partitioned tables.
 
 ## Declarative Partitioning
 
@@ -296,7 +334,31 @@ Postgres 10 introduces [declarative partitioning](https://www.postgresql.org/doc
 
 ## Data Protection
 
-Always make sure your connection is secure when connecting to your database over a network you don’t fully trust. Your best option is to connect over SSH or a VPN. Another option is to use `sslmode=verify-full`. If you don’t do this, your database credentials can be compromised.
+Always make sure your [connection is secure](https://ankane.org/postgres-sslmode-explained) when connecting to a database over a network you don’t fully trust. Your best option is to connect over SSH or a VPN. Another option is to use `sslmode=verify-full`. If you don’t do this, your database credentials can be compromised.
+
+## Homebrew
+
+On Mac, you can use:
+
+```sh
+brew install ankane/brew/pgslice
+```
+
+## Dependencies
+
+If installation fails, your system may be missing Ruby or libpq.
+
+On Mac, run:
+
+```sh
+brew install postgresql
+```
+
+On Ubuntu, run:
+
+```sh
+sudo apt-get install ruby-dev libpq-dev build-essential
+```
 
 ## Upgrading
 
@@ -331,6 +393,7 @@ This will give you the `pgslice` command.
 
 Also check out:
 
+- [Dexter](https://github.com/ankane/dexter) - The automatic indexer for Postgres
 - [PgHero](https://github.com/ankane/pghero) - A performance dashboard for Postgres
 - [pgsync](https://github.com/ankane/pgsync) - Sync Postgres data to your local machine
 
@@ -343,14 +406,14 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 - Write, clarify, or fix documentation
 - Suggest or add new features
 
-To run tests, do:
+To get started with development:
 
 ```sh
 git clone https://github.com/ankane/pgslice.git
 cd pgslice
 bundle install
 createdb pgslice_test
-bundle exec rake
+bundle exec rake test
 ```
 
 To test against different versions of Postgres with Docker, use:

data/exe/pgslice

@@ -1,8 +1,7 @@
 #!/usr/bin/env ruby
 
+# handle interrupts
+trap("SIGINT") { abort }
+
 require "pgslice"
-begin
-  PgSlice::CLI.start
-rescue Interrupt
-  abort
-end
+PgSlice::CLI.start

data/lib/pgslice/cli/add_partitions.rb

@@ -4,6 +4,7 @@ module PgSlice
     option :intermediate, type: :boolean, default: false, desc: "Add to intermediate table"
     option :past, type: :numeric, default: 0, desc: "Number of past partitions to add"
     option :future, type: :numeric, default: 0, desc: "Number of future partitions to add"
+    option :tablespace, type: :string, default: "", desc: "Tablespace to use"
     def add_partitions(table)
       original_table = create_table(table)
       table = options[:intermediate] ? original_table.intermediate_table : original_table
@@ -13,6 +14,7 @@ module PgSlice
 
       future = options[:future]
       past = options[:past]
+      tablespace = options[:tablespace]
       range = (-1 * past)..future
 
       period, field, cast, needs_comment, declarative, version = table.fetch_settings(original_table.trigger_name)
@@ -50,6 +52,7 @@ module PgSlice
       end
 
       primary_key = schema_table.primary_key
+      tablespace_str = tablespace.empty? ? "" : " TABLESPACE #{quote_ident(tablespace)}"
 
       added_partitions = []
       range.each do |n|
@@ -61,13 +64,13 @@ module PgSlice
 
         if declarative
           queries << <<-SQL
-CREATE TABLE #{quote_table(partition)} PARTITION OF #{quote_table(table)} FOR VALUES FROM (#{sql_date(day, cast, false)}) TO (#{sql_date(advance_date(day, period, 1), cast, false)});
+CREATE TABLE #{quote_table(partition)} PARTITION OF #{quote_table(table)} FOR VALUES FROM (#{sql_date(day, cast, false)}) TO (#{sql_date(advance_date(day, period, 1), cast, false)})#{tablespace_str};
           SQL
         else
           queries << <<-SQL
 CREATE TABLE #{quote_table(partition)}
     (CHECK (#{quote_ident(field)} >= #{sql_date(day, cast)} AND #{quote_ident(field)} < #{sql_date(advance_date(day, period, 1), cast)}))
-    INHERITS (#{quote_table(table)});
+    INHERITS (#{quote_table(table)})#{tablespace_str};
           SQL
         end
 

data/lib/pgslice/cli/swap.rb

@@ -17,7 +17,7 @@ module PgSlice
       ]
 
       table.sequences.each do |sequence|
-        queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_name"])} OWNED BY #{quote_table(table)}.#{quote_ident(sequence["related_column"])};"
+        queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_schema"])}.#{quote_ident(sequence["sequence_name"])} OWNED BY #{quote_table(table)}.#{quote_ident(sequence["related_column"])};"
       end
 
       queries.unshift("SET LOCAL lock_timeout = '#{options[:lock_timeout]}';") if server_version_num >= 90300

data/lib/pgslice/cli/unswap.rb

@@ -16,7 +16,7 @@ module PgSlice
       ]
 
       table.sequences.each do |sequence|
-        queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_name"])} OWNED BY #{quote_table(table)}.#{quote_ident(sequence["related_column"])};"
+        queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_schema"])}.#{quote_ident(sequence["sequence_name"])} OWNED BY #{quote_table(table)}.#{quote_ident(sequence["related_column"])};"
       end
 
       run_queries(queries)

data/lib/pgslice/cli.rb

@@ -14,7 +14,7 @@ module PgSlice
     map %w[--version -v] => :version
 
     def self.exit_on_failure?
-      true
+      ENV["PGSLICE_ENV"] != "test"
     end
 
     def initialize(*args)

data/lib/pgslice/helpers.rb

@@ -33,10 +33,14 @@ module PgSlice
         params = CGI.parse(uri.query.to_s)
         # remove schema
         @schema = Array(params.delete("schema") || "public")[0]
-        uri.query = URI.encode_www_form(params)
+        uri.query = params.any? ? URI.encode_www_form(params) : nil
 
         ENV["PGCONNECT_TIMEOUT"] ||= "1"
-        PG::Connection.new(uri.to_s)
+        conn = PG::Connection.new(uri.to_s)
+        conn.set_notice_processor do |message|
+          say message
+        end
+        conn
       end
     rescue PG::ConnectionBad => e
       abort e.message

data/lib/pgslice/table.rb

@@ -23,16 +23,19 @@ module PgSlice
     def sequences
       query = <<-SQL
         SELECT
-          a.attname as related_column,
-          s.relname as sequence_name
+          a.attname AS related_column,
+          n.nspname AS sequence_schema,
+          s.relname AS sequence_name
         FROM pg_class s
-          JOIN pg_depend d ON d.objid = s.oid
-          JOIN pg_class t ON d.objid = s.oid AND d.refobjid = t.oid
-          JOIN pg_attribute a ON (d.refobjid, d.refobjsubid) = (a.attrelid, a.attnum)
-          JOIN pg_namespace n ON n.oid = s.relnamespace
+          INNER JOIN pg_depend d ON d.objid = s.oid
+          INNER JOIN pg_class t ON d.objid = s.oid AND d.refobjid = t.oid
+          INNER JOIN pg_attribute a ON (d.refobjid, d.refobjsubid) = (a.attrelid, a.attnum)
+          INNER JOIN pg_namespace n ON n.oid = s.relnamespace
+          INNER JOIN pg_namespace nt ON nt.oid = t.relnamespace
         WHERE s.relkind = 'S'
-          AND n.nspname = $1
+          AND nt.nspname = $1
           AND t.relname = $2
+        ORDER BY s.relname ASC
       SQL
       execute(query, [schema, name])
     end
@@ -41,12 +44,15 @@ module PgSlice
       execute("SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid = #{regclass} AND contype ='f'").map { |r| r["pg_get_constraintdef"] }
     end
 
-    # http://stackoverflow.com/a/20537829
+    # https://stackoverflow.com/a/20537829
+    # TODO can simplify with array_position in Postgres 9.5+
     def primary_key
       query = <<-SQL
         SELECT
           pg_attribute.attname,
-          format_type(pg_attribute.atttypid, pg_attribute.atttypmod)
+          format_type(pg_attribute.atttypid, pg_attribute.atttypmod),
+          pg_attribute.attnum,
+          pg_index.indkey
         FROM
           pg_index, pg_class, pg_attribute, pg_namespace
         WHERE
@@ -58,7 +64,8 @@ module PgSlice
           pg_attribute.attnum = any(pg_index.indkey) AND
           indisprimary
       SQL
-      execute(query, [schema, name]).map { |r| r["attname"] }
+      rows = execute(query, [schema, name])
+      rows.sort_by { |r| r["indkey"].split(" ").index(r["attnum"]) }.map { |r| r["attname"] }
     end
 
     def index_defs
@@ -104,6 +111,7 @@ module PgSlice
       (execute(query)[0]["min"] || 1).to_i
     end
 
+    # ensure this returns partitions in the correct order
     def partitions
       query = <<-SQL
         SELECT
@@ -117,6 +125,7 @@ module PgSlice
         WHERE
           nmsp_parent.nspname = $1 AND
           parent.relname = $2
+        ORDER BY child.relname ASC
       SQL
       execute(query, [schema, name]).map { |r| Table.new(r["schema"], r["name"]) }
     end
@@ -144,7 +153,7 @@ module PgSlice
         function_def = execute("SELECT pg_get_functiondef(oid) FROM pg_proc WHERE proname = $1", [trigger_name])[0]
         return [] unless function_def
         function_def = function_def["pg_get_functiondef"]
-        sql_format = SQL_FORMAT.find { |_, f| function_def.include?("'#{f}'") }
+        sql_format = Helpers::SQL_FORMAT.find { |_, f| function_def.include?("'#{f}'") }
         return [] unless sql_format
         period = sql_format[0]
         field = /to_char\(NEW\.(\w+),/.match(function_def)[1]

data/lib/pgslice/version.rb

@@ -1,3 +1,3 @@
 module PgSlice
-  VERSION = "0.4.5"
+  VERSION = "0.4.8"
 end

data/lib/pgslice.rb

@@ -1,7 +1,9 @@
 # dependencies
-require "cgi"
 require "pg"
 require "thor"
+
+# stdlib
+require "cgi"
 require "time"
 
 # modules

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgslice
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.4.8
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2018-10-18 00:00:00.000000000 Z
+date: 2022-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -38,50 +38,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.18.2
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: 
-email: andrew@chartkick.com
+description:
+email: andrew@ankane.org
 executables:
 - pgslice
 extensions: []
@@ -107,7 +65,7 @@ homepage: https://github.com/ankane/pgslice
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -122,9 +80,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
-signing_key: 
+rubygems_version: 3.3.3
+signing_key:
 specification_version: 4
 summary: Postgres partitioning as easy as pie
 test_files: []