pgslice 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 10c9e21963650fa68dd402f3c690ab0b89300bfb
-  data.tar.gz: 5fe1cea0ef611008c95dbf06eae19e766dbf679e
+  metadata.gz: 1e344dce1fde075ab0621622d3f929e2c6677026
+  data.tar.gz: 9fe6e7825852fb37692ff019908dfb82e5cf4095
 SHA512:
-  metadata.gz: 6f879dec8f413fd90eed3f911aacaedf5dda87036d490cab18d14fb1447439114a2359db4f15a95e3f713bcd0bf2bcad10f65c96c69537b83ed4172820b2d582
-  data.tar.gz: 74189c73cdd2e6cf2549b31524c9e64103719d78135708bc2d992a118a0413a1b6d63cc4dc22659427717938fde3be5b8caab53d17af6d3088628697cc94256e
+  metadata.gz: a684bbbbc65be40c4a42850f0b54af657371649efef72f9f89337a8fae4ac046bd1fe758f1632ff2fc15c3a3d1e2c6feec7f2eaa70e13d9f815565993ff0e85a
+  data.tar.gz: 6b5e6fe5958f81085c8ec9a3d0d0161e35aa29f616b2db32dca993b520e14178face97dbe43133dc919b750b24a220cfe5819cd1732f6c64307cf81f6ee22363

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.2.1
+
+- Added `--where` option to `fill`
+- Fixed partition detection with `fill`
+- Fixed error for columns named `user` with `fill`
+
 ## 0.2.0
 
 - Switched to new trigger, which is about 20% faster

data/README.md

@@ -1,6 +1,6 @@
 # pgslice
 
-Postgres partitioning as easy as pie. Works great for both new and existing tables, with zero downtime and minimal app changes.
+Postgres partitioning as easy as pie. Works great for both new and existing tables, with zero downtime and minimal app changes. Archive older data on a rolling basis to keep your database size under control.
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
@@ -68,7 +68,12 @@ This will give you the `pgslice` command.
   pgslice fill <table> --swapped
   ```
 
-8. Archive and drop the original table
+8. Back up the retired table with a tool like [pg_dump](https://www.postgresql.org/docs/current/static/app-pgdump.html) and drop it
+
+  ```sql
+  pg_dump -c -Fc -t <table>_retired $PGSLICE_URL > <table>_retired.dump
+  psql -c "DROP <table>_retired" $PGSLICE_URL
+  ```
 
 ## Sample Output
 
@@ -86,7 +91,7 @@ CREATE TABLE visits_intermediate (LIKE visits INCLUDING ALL);
 CREATE FUNCTION visits_insert_trigger()
     RETURNS trigger AS $$
     BEGIN
-        RAISE EXCEPTION 'Date out of range. Create partitions first.';
+        RAISE EXCEPTION 'Create partitions first.';
     END;
     $$ LANGUAGE plpgsql;
 
@@ -155,18 +160,18 @@ pgslice fill visits
 
 ```sql
 /* 1 of 3 */
-INSERT INTO visits_intermediate (id, user_id, ip, created_at)
-    SELECT id, user_id, ip, created_at FROM visits
+INSERT INTO visits_intermediate ("id", "user_id", "ip", "created_at")
+    SELECT "id", "user_id", "ip", "created_at" FROM visits
     WHERE id > 0 AND id <= 10000 AND created_at >= '2016-08-01'::date AND created_at < '2016-11-01'::date
 
 /* 2 of 3 */
-INSERT INTO visits_intermediate (id, user_id, ip, created_at)
-    SELECT id, user_id, ip, created_at FROM visits
+INSERT INTO visits_intermediate ("id", "user_id", "ip", "created_at")
+    SELECT "id", "user_id", "ip", "created_at" FROM visits
     WHERE id > 10000 AND id <= 20000 AND created_at >= '2016-08-01'::date AND created_at < '2016-11-01'::date
 
 /* 3 of 3 */
-INSERT INTO visits_intermediate (id, user_id, ip, created_at)
-    SELECT id, user_id, ip, created_at FROM visits
+INSERT INTO visits_intermediate ("id", "user_id", "ip", "created_at")
+    SELECT "id", "user_id", "ip", "created_at" FROM visits
     WHERE id > 20000 AND id <= 30000 AND created_at >= '2016-08-01'::date AND created_at < '2016-11-01'::date
 ```
 
@@ -218,6 +223,21 @@ WHERE
     -- for months, use to_char(NOW() + INTERVAL '3 months', 'YYYYMM')
 ```
 
+## Archiving Partitions
+
+Back up and drop older partitions each day or month.
+
+```sh
+pg_dump -c -Fc -t <table>_201609 $PGSLICE_URL > <table>_201609.dump
+psql -c "DROP <table>_201609" $PGSLICE_URL
+```
+
+If you use [Amazon S3](https://aws.amazon.com/s3/) for backups, [s3cmd](https://github.com/s3tools/s3cmd) is a nice tool.
+
+```sh
+s3cmd put <table>_201609.dump s3://<s3-bucket>/<table>_201609.dump
+```
+
 ## Additional Commands
 
 To undo prep (which will delete partitions), use:
@@ -268,7 +288,7 @@ You can also use pgslice to reduce the size of a table without partitioning by c
 
 ```sh
 pgslice prep <table> --no-partition
-pgslice fill <table> --start 1000 # starting primary key
+pgslice fill <table> --where "id > 1000" # use any conditions
 pgslice swap <table>
 ```
 

data/lib/pgslice.rb

@@ -79,7 +79,7 @@ CREATE TABLE #{intermediate_table} (LIKE #{table} INCLUDING ALL);
 CREATE FUNCTION #{trigger_name}()
     RETURNS trigger AS $$
     BEGIN
-        RAISE EXCEPTION 'Date out of range. Create partitions first.';
+        RAISE EXCEPTION 'Create partitions first.';
     END;
     $$ LANGUAGE plpgsql;
         SQL
@@ -133,16 +133,13 @@ SQL
 
       queries = []
 
-      comment = execute("SELECT obj_description(oid, 'pg_trigger') AS comment FROM pg_trigger WHERE tgname = $1 AND tgrelid = $2::regclass", [trigger_name, table]).first
-      if comment
-        field, period = comment["comment"].split(",").map { |v| v.split(":").last } rescue [nil, nil]
-      end
+      period, field, needs_comment = settings_from_trigger(original_table, table)
+      abort "Could not read settings" unless period
 
-      unless period
-        period, field = settings_from_table(original_table, table)
+      if needs_comment
         queries << "COMMENT ON TRIGGER #{trigger_name} ON #{table} is 'column:#{field},period:#{period}';"
       end
-      abort "Could not read settings" unless period
+
       # today = utc date
       today = round_date(DateTime.now.new_offset(0).to_date, period)
       added_partitions = []
@@ -229,7 +226,7 @@ CREATE OR REPLACE FUNCTION #{trigger_name}()
       abort "Table not found: #{source_table}" unless table_exists?(source_table)
       abort "Table not found: #{dest_table}" unless table_exists?(dest_table)
 
-      period, field = settings_from_table(table, dest_table)
+      period, field, needs_comment = settings_from_trigger(table, dest_table)
 
       if period
         name_format = self.name_format(period)
@@ -258,7 +255,7 @@ CREATE OR REPLACE FUNCTION #{trigger_name}()
       end
 
       starting_id = max_dest_id
-      fields = columns(source_table).join(", ")
+      fields = columns(source_table).map { |c| PG::Connection.quote_ident(c) }.join(", ")
       batch_size = options[:batch_size]
 
       i = 1
@@ -268,6 +265,9 @@ CREATE OR REPLACE FUNCTION #{trigger_name}()
         if period
           where << " AND #{field} >= #{sql_date(starting_time)} AND #{field} < #{sql_date(ending_time)}"
         end
+        if options[:where]
+          where << " AND #{options[:where]}"
+        end
 
         query = <<-SQL
 /* #{i} of #{batch_count} */
@@ -348,6 +348,7 @@ INSERT INTO #{dest_table} (#{fields})
         o.integer "--start"
         o.string "--url"
         o.string "--source-table"
+        o.string "--where"
         o.on "-v", "--version", "print the version" do
           log PgSlice::VERSION
           @exit = true
@@ -527,16 +528,27 @@ INSERT INTO #{dest_table} (#{fields})
       end
     end
 
-    def settings_from_table(original_table, table)
+    def settings_from_trigger(original_table, table)
       trigger_name = self.trigger_name(original_table)
-      function_def = execute("select pg_get_functiondef(oid) from pg_proc where proname = $1", [trigger_name])[0]
-      return [nil, nil] unless function_def
-      function_def = function_def["pg_get_functiondef"]
-      sql_format = SQL_FORMAT.find { |_, f| function_def.include?("'#{f}'") }
-      return [nil, nil] unless sql_format
-      period = sql_format[0]
-      field = /to_char\(NEW\.(\w+),/.match(function_def)[1]
-      [period, field]
+
+      needs_comment = false
+      comment = execute("SELECT obj_description(oid, 'pg_trigger') AS comment FROM pg_trigger WHERE tgname = $1 AND tgrelid = $2::regclass", [trigger_name, table])[0]
+      if comment
+        field, period = comment["comment"].split(",").map { |v| v.split(":").last } rescue [nil, nil]
+      end
+
+      unless period
+        needs_comment = true
+        function_def = execute("select pg_get_functiondef(oid) from pg_proc where proname = $1", [trigger_name])[0]
+        return [nil, nil] unless function_def
+        function_def = function_def["pg_get_functiondef"]
+        sql_format = SQL_FORMAT.find { |_, f| function_def.include?("'#{f}'") }
+        return [nil, nil] unless sql_format
+        period = sql_format[0]
+        field = /to_char\(NEW\.(\w+),/.match(function_def)[1]
+      end
+
+      [period, field, needs_comment]
     end
   end
 end

data/lib/pgslice/version.rb

@@ -1,3 +1,3 @@
 module PgSlice
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgslice
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-09-23 00:00:00.000000000 Z
+date: 2016-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop