pgslice 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4e0437c8314b62daec17ddf072a9f269fa6b7aa1
-  data.tar.gz: 311105b8057bc43cf9ff75a7a8be0dc62b707c9b
+  metadata.gz: 1fbf523156daf638be0cf5de6a77c4c2ed4375ab
+  data.tar.gz: fa1839b58db3046ab8e4705a410c7c7e3d99a86a
 SHA512:
-  metadata.gz: 4db509fb649e78d0fac104071e58621b54a01bc5b17b53bbd5aa786ee9908e657ed6adaf6e407ffae28cfc81549cdde76512e4a6b9195a18ac421ac31eb21755
-  data.tar.gz: a43f16632bca4827762396778d3ed23718046f774e584ef65b6a48af8245d195014c4f628a190f9e58c0d534b4bd9b8aaf4dd59d776d4428e9f56afdaacc69ae
+  metadata.gz: 697401d7e40ad0db4f7be59d2a0e711b8f1e57c7c3efdf8874d4ed29f915bf5e98bc967e6f6199990a53853796e214f21cfbfe0961bd504defa8143124f0163a
+  data.tar.gz: bc09c5daebc448a40aace451e80733a15f5d1ef6443e7f3fb7a5d7d82f3fee444b4e918c9b08dd6f80ec82a956ab5882a11d5997438025e4d394a0fe0b5b89b3

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.3.0
+
+- Better query performance for `timestamptz` columns
+- Added support for schemas other than `public`
+
 ## 0.2.3
 
 - Added `--dest-table` option to `fill`

data/Dockerfile

@@ -0,0 +1,3 @@
+FROM ruby:latest
+RUN gem install pgslice
+ENTRYPOINT ["pgslice"]

data/README.md

@@ -254,10 +254,17 @@ To undo swap, use:
 pgslice unswap <table>
 ```
 
-## App Changes
+## App Considerations
 
 This set up allows you to read and write with the original table name with no knowledge it’s partitioned. However, there are a few things to be aware of.
 
+### Writes
+
+If you use `INSERT` statements with a `RETURNING` clause (as frameworks like Rails do), you’ll no longer receive the id of the newly inserted record(s) back. If you need this, you can either:
+
+1. Insert directly into the partition
+2. Get the value after the insert with `SELECT CURRVAL('sequence_name')`
+
 ### Reads
 
 When possible, queries should include the column you partition on to limit the number of partitions the database needs to check.  For instance, if you partition on `created_at`, try to include it in queries:
@@ -277,13 +284,6 @@ For this to be effective, ensure `constraint_exclusion` is set to `partition` (d
 SHOW constraint_exclusion;
 ```
 
-### Writes
-
-If you use `INSERT` statements with a `RETURNING` clause (as frameworks like Rails do), you’ll no longer receive the id of the newly inserted record back. If you need this, you can either:
-
-1. Insert directly into the partition
-2. Get the value after the insert with `SELECT CURRVAL('sequence_name')`
-
 ## One Off Tasks
 
 You can also use pgslice to reduce the size of a table without partitioning by creating a new table, filling it with a subset of records, and swapping it in.
@@ -309,6 +309,15 @@ gem install specific_install
 gem specific_install ankane/pgslice
 ```
 
+## Docker
+
+```sh
+docker build -t pgslice .
+alias pgslice="docker run --rm -e PGSLICE_URL pgslice"
+```
+
+This will give you the `pgslice` command.
+
 ## Reference
 
 - [PostgreSQL Manual](https://www.postgresql.org/docs/current/static/ddl-partitioning.html)

data/lib/pgslice.rb

@@ -1,6 +1,7 @@
 require "pgslice/version"
 require "slop"
 require "pg"
+require "cgi"
 
 module PgSlice
   class Error < StandardError; end
@@ -51,6 +52,7 @@ module PgSlice
 
     def prep
       table, column, period = arguments
+      cast = column_cast(table, column)
       intermediate_table = "#{table}_intermediate"
       trigger_name = self.trigger_name(table)
 
@@ -91,7 +93,7 @@ CREATE TRIGGER #{trigger_name}
       SQL
 
         queries << <<-SQL
-COMMENT ON TRIGGER #{trigger_name} ON #{intermediate_table} is 'column:#{column},period:#{period}';
+COMMENT ON TRIGGER #{trigger_name} ON #{intermediate_table} is 'column:#{column},period:#{period},cast:#{cast}';
 SQL
       end
 
@@ -133,11 +135,11 @@ SQL
 
       queries = []
 
-      period, field, needs_comment = settings_from_trigger(original_table, table)
+      period, field, cast, needs_comment = settings_from_trigger(original_table, table)
       abort "Could not read settings" unless period
 
       if needs_comment
-        queries << "COMMENT ON TRIGGER #{trigger_name} ON #{table} is 'column:#{field},period:#{period}';"
+        queries << "COMMENT ON TRIGGER #{trigger_name} ON #{table} is 'column:#{field},period:#{period},cast:#{cast}';"
       end
 
       # today = utc date
@@ -152,7 +154,7 @@ SQL
 
         queries << <<-SQL
 CREATE TABLE #{partition_name}
-    (CHECK (#{field} >= #{sql_date(day)} AND #{field} < #{sql_date(advance_date(day, period, 1))}))
+    (CHECK (#{field} >= #{sql_date(day, cast)} AND #{field} < #{sql_date(advance_date(day, period, 1), cast)}))
     INHERITS (#{table});
         SQL
 
@@ -175,7 +177,7 @@ CREATE TABLE #{partition_name}
         day = DateTime.strptime(table.split("_").last, name_format)
         partition_name = "#{original_table}_#{day.strftime(name_format(period))}"
 
-        sql = "(NEW.#{field} >= #{sql_date(day)} AND NEW.#{field} < #{sql_date(advance_date(day, period, 1))}) THEN
+        sql = "(NEW.#{field} >= #{sql_date(day, cast)} AND NEW.#{field} < #{sql_date(advance_date(day, period, 1), cast)}) THEN
             INSERT INTO #{partition_name} VALUES (NEW.*);"
 
         if day.to_date < today
@@ -227,7 +229,7 @@ CREATE OR REPLACE FUNCTION #{trigger_name}()
       abort "Table not found: #{source_table}" unless table_exists?(source_table)
       abort "Table not found: #{dest_table}" unless table_exists?(dest_table)
 
-      period, field, needs_comment = settings_from_trigger(table, dest_table)
+      period, field, cast, needs_comment = settings_from_trigger(table, dest_table)
 
       if period
         name_format = self.name_format(period)
@@ -252,7 +254,7 @@ CREATE OR REPLACE FUNCTION #{trigger_name}()
         if options[:start]
           max_dest_id = options[:start]
         else
-          min_source_id = min_id(source_table, primary_key, field, starting_time, options[:where])
+          min_source_id = min_id(source_table, primary_key, field, cast, starting_time, options[:where])
           max_dest_id = min_source_id - 1 if min_source_id
         end
       end
@@ -260,13 +262,14 @@ CREATE OR REPLACE FUNCTION #{trigger_name}()
       starting_id = max_dest_id
       fields = columns(source_table).map { |c| PG::Connection.quote_ident(c) }.join(", ")
       batch_size = options[:batch_size]
+      cast = column_cast(table, field)
 
       i = 1
       batch_count = ((max_source_id - starting_id) / batch_size.to_f).ceil
       while starting_id < max_source_id
         where = "#{primary_key} > #{starting_id} AND #{primary_key} <= #{starting_id + batch_size}"
         if starting_time
-          where << " AND #{field} >= #{sql_date(starting_time)} AND #{field} < #{sql_date(ending_time)}"
+          where << " AND #{field} >= #{sql_date(starting_time, cast)} AND #{field} < #{sql_date(ending_time, cast)}"
         end
         if options[:where]
           where << " AND #{options[:where]}"
@@ -398,8 +401,16 @@ INSERT INTO #{dest_table} (#{fields})
           connect_timeout: 1
         }.reject { |_, value| value.to_s.empty? }
         config.map { |key, value| config[key] = uri_parser.unescape(value) if value.is_a?(String) }
+        @schema = CGI.parse(uri.query.to_s)["schema"][0] || "public"
         PG::Connection.new(config)
       end
+    rescue PG::ConnectionBad => e
+      abort e.message
+    end
+
+    def schema
+      connection # ensure called first
+      @schema
     end
 
     def execute(query, params = [])
@@ -432,7 +443,7 @@ INSERT INTO #{dest_table} (#{fields})
 
     def existing_tables(like:)
       query = "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = $1 AND tablename LIKE $2"
-      execute(query, ["public", like]).map { |r| r["tablename"] }.sort
+      execute(query, [schema, like]).map { |r| r["tablename"] }.sort
     end
 
     def table_exists?(table)
@@ -440,7 +451,7 @@ INSERT INTO #{dest_table} (#{fields})
     end
 
     def columns(table)
-      execute("SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = $1", [table]).map{ |r| r["column_name"] }
+      execute("SELECT column_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2", [schema, table]).map{ |r| r["column_name"] }
     end
 
     # http://stackoverflow.com/a/20537829
@@ -460,7 +471,7 @@ INSERT INTO #{dest_table} (#{fields})
           pg_attribute.attnum = any(pg_index.indkey) AND
           indisprimary
       SQL
-      row = execute(query, ["public", table])[0]
+      row = execute(query, [schema, table])[0]
       row && row["attname"]
     end
 
@@ -473,10 +484,11 @@ INSERT INTO #{dest_table} (#{fields})
       execute(query)[0]["max"].to_i
     end
 
-    def min_id(table, primary_key, column, starting_time, where)
+    def min_id(table, primary_key, column, cast, starting_time, where)
+      cast = column_cast(table, column)
       query = "SELECT MIN(#{primary_key}) FROM #{table}"
       conditions = []
-      conditions << "#{column} >= #{sql_date(starting_time)}" if starting_time
+      conditions << "#{column} >= #{sql_date(starting_time, cast)}" if starting_time
       conditions << where if where
       query << " WHERE #{conditions.join(" AND ")}" if conditions.any?
       (execute(query)[0]["min"] || 1).to_i
@@ -498,10 +510,10 @@ INSERT INTO #{dest_table} (#{fields})
           JOIN pg_attribute a ON (d.refobjid, d.refobjsubid) = (a.attrelid, a.attnum)
           JOIN pg_namespace n ON n.oid = s.relnamespace
         WHERE s.relkind = 'S'
-          AND n.nspname = 'public'
-          AND t.relname = $1
+          AND n.nspname = $1
+          AND t.relname = $2
       SQL
-      execute(query, [table])
+      execute(query, [schema, table])
     end
 
     # helpers
@@ -518,8 +530,18 @@ INSERT INTO #{dest_table} (#{fields})
       "#{table}_retired"
     end
 
-    def sql_date(time)
-      "'#{time.strftime("%Y-%m-%d")}'::date"
+    def column_cast(table, column)
+      data_type = execute("SELECT data_type FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 AND column_name = $3", [schema, table, column])[0]["data_type"]
+      data_type == "timestamp with time zone" ? "timestamptz" : "date"
+    end
+
+    def sql_date(time, cast)
+      if cast == "timestamptz"
+        fmt = "%Y-%m-%d %H:%M:%S UTC"
+      else
+        fmt = "%Y-%m-%d"
+      end
+      "'#{time.strftime(fmt)}'::#{cast}"
     end
 
     def name_format(period)
@@ -557,7 +579,7 @@ INSERT INTO #{dest_table} (#{fields})
       needs_comment = false
       comment = execute("SELECT obj_description(oid, 'pg_trigger') AS comment FROM pg_trigger WHERE tgname = $1 AND tgrelid = $2::regclass", [trigger_name, table])[0]
       if comment
-        field, period = comment["comment"].split(",").map { |v| v.split(":").last } rescue [nil, nil]
+        field, period, cast = comment["comment"].split(",").map { |v| v.split(":").last } rescue [nil, nil, nil]
       end
 
       unless period
@@ -571,7 +593,14 @@ INSERT INTO #{dest_table} (#{fields})
         field = /to_char\(NEW\.(\w+),/.match(function_def)[1]
       end
 
-      [period, field, needs_comment]
+      # backwards compatibility with 0.2.3 and earlier (pre-timestamptz support)
+      unless cast
+        cast = "date"
+        # update comment to explicitly define cast
+        needs_comment = true
+      end
+
+      [period, field, cast, needs_comment]
     end
   end
 end

data/lib/pgslice/version.rb

@@ -1,3 +1,3 @@
 module PgSlice
-  VERSION = "0.2.3"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgslice
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-11 00:00:00.000000000 Z
+date: 2016-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -76,6 +76,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - CHANGELOG.md
+- Dockerfile
 - Gemfile
 - README.md
 - Rakefile