pgslice 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 27fdf533604b2635d75b2eb92043617f8a53ed7a
-  data.tar.gz: 5f5579762fdfe759f38a4eb03cce91acb90a7397
+  metadata.gz: 63369b304364c5cbc5b6b96e9303d3f46293d0b3
+  data.tar.gz: 15e76add4582111650e46daad02bbe755b3e45e1
 SHA512:
-  metadata.gz: ac1abb9261c93e8b0bf057b6b16fd1a864f3e2ddd9b85f196df095731fbbbdac0b49e3878b966632c15ce9d7c6774d64231dc03d2e4a42b74fd185ffc177a8b5
-  data.tar.gz: 9524851474990255e54c008477db65de535fcea6412900ea0c68a801518a79b7124175cde7cddd5e932a6575c76d282047642d7c8f4016e6cce13fd703a90877
+  metadata.gz: 64d23629b73f8adf89ce6adc03c2cc951b9b67e7edc8e03eb596546e43b40f1415ea494ccdcab4efb460686ff967941ddf4ba66491525d3b4612490b017b4a0f
+  data.tar.gz: 864e600e79f944667c11a151a23341366085b39d1cf81b10f1ef03408c9d6101690ad3697d36369f0151353baa90d7f7d0429b0507c6d738c33396ffb3cefa47

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.1.4
+
+- Added sequence ownership
+- Default to 0 for `--past` and `--future` options
+- Better `fill` with `--swapped`
+
 ## 0.1.3
 
 - Fixed table inheritance

data/README.md

@@ -92,21 +92,19 @@ pgslice unswap <table>
 $ pgslice prep locations created_at day
 BEGIN;
 
-CREATE TABLE locations_intermediate (
-  LIKE locations INCLUDING ALL
-);
+CREATE TABLE locations_intermediate (LIKE locations INCLUDING ALL);
 
 CREATE FUNCTION locations_insert_trigger()
-RETURNS trigger AS $$
-BEGIN
-  EXECUTE 'INSERT INTO public.locations_' || to_char(NEW.created_at, 'YYYYMMDD') || ' VALUES ($1.*)' USING NEW;
-  RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
+    RETURNS trigger AS $$
+    BEGIN
+        EXECUTE 'INSERT INTO locations_' || to_char(NEW.created_at, 'YYYYMMDD') || ' VALUES ($1.*)' USING NEW;
+        RETURN NULL;
+    END;
+    $$ LANGUAGE plpgsql;
 
 CREATE TRIGGER locations_insert_trigger
-BEFORE INSERT ON locations_intermediate
-FOR EACH ROW EXECUTE PROCEDURE locations_insert_trigger();
+    BEFORE INSERT ON locations_intermediate
+    FOR EACH ROW EXECUTE PROCEDURE locations_insert_trigger();
 
 COMMIT;
 ```
@@ -115,25 +113,25 @@ COMMIT;
 $ pgslice add_partitions locations --intermediate --past 1 --future 1
 BEGIN;
 
-CREATE TABLE locations_20160423 (
-  CHECK (created_at >= '2016-04-23'::date AND created_at < '2016-04-24'::date)
-) INHERITS (locations_intermediate);
+CREATE TABLE locations_20160423
+    (CHECK (created_at >= '2016-04-23'::date AND created_at < '2016-04-24'::date))
+    INHERITS (locations_intermediate);
 
 ALTER TABLE locations_20160423 ADD PRIMARY KEY (id);
 
 CREATE INDEX ON locations_20160423 USING btree (updated_at, shopper_id);
 
-CREATE TABLE locations_20160424 (
-  CHECK (created_at >= '2016-04-24'::date AND created_at < '2016-04-25'::date)
-) INHERITS (locations_intermediate);
+CREATE TABLE locations_20160424
+    (CHECK (created_at >= '2016-04-24'::date AND created_at < '2016-04-25'::date))
+    INHERITS (locations_intermediate);
 
 ALTER TABLE locations_20160424 ADD PRIMARY KEY (id);
 
 CREATE INDEX ON locations_20160424 USING btree (updated_at, shopper_id);
 
-CREATE TABLE locations_20160425 (
-  CHECK (created_at >= '2016-04-25'::date AND created_at < '2016-04-26'::date)
-) INHERITS (locations_intermediate);
+CREATE TABLE locations_20160425
+    (CHECK (created_at >= '2016-04-25'::date AND created_at < '2016-04-26'::date))
+    INHERITS (locations_intermediate);
 
 ALTER TABLE locations_20160425 ADD PRIMARY KEY (id);
 
@@ -142,6 +140,24 @@ CREATE INDEX ON locations_20160425 USING btree (updated_at, shopper_id);
 COMMIT;
 ```
 
+```console
+$ pgslice fill locations
+/* 1 of 3 */
+INSERT INTO locations_intermediate (id, latitude, longitude, created_at)
+    SELECT id, latitude, longitude, created_at FROM locations
+    WHERE id > 0 AND id <= 10000 AND created_at >= '2016-04-23'::date AND created_at < '2016-04-26'::date
+
+/* 2 of 3 */
+INSERT INTO locations_intermediate (id, latitude, longitude, created_at)
+    SELECT id, latitude, longitude, created_at FROM locations
+    WHERE id > 10000 AND id <= 20000 AND created_at >= '2016-04-23'::date AND created_at < '2016-04-26'::date
+
+/* 3 of 3 */
+INSERT INTO locations_intermediate (id, latitude, longitude, created_at)
+    SELECT id, latitude, longitude, created_at FROM locations
+    WHERE id > 20000 AND id <= 30000 AND created_at >= '2016-04-23'::date AND created_at < '2016-04-26'::date
+```
+
 ```console
 $ pgslice swap locations
 BEGIN;
@@ -150,6 +166,23 @@ ALTER TABLE locations RENAME TO locations_retired;
 
 ALTER TABLE locations_intermediate RENAME TO locations;
 
+ALTER SEQUENCE locations_id_seq OWNED BY locations.id;
+
+COMMIT;
+```
+
+```console
+$ pgslice add_partitions locations --future 2
+BEGIN;
+
+CREATE TABLE locations_20160426
+    (CHECK (created_at >= '2016-04-26'::date AND created_at < '2016-04-27'::date))
+    INHERITS (locations);
+
+ALTER TABLE locations_20160426 ADD PRIMARY KEY (id);
+
+CREATE INDEX ON locations_20160426 USING btree (updated_at, shopper_id);
+
 COMMIT;
 ```
 

data/lib/pgslice.rb

@@ -15,6 +15,8 @@ module PgSlice
     }
 
     def initialize(args)
+      $stdout.sync = true
+      $stderr.sync = true
       parse_args(args)
       @command = @arguments.shift
     end
@@ -62,26 +64,24 @@ module PgSlice
       queries = []
 
       queries << <<-SQL
-CREATE TABLE #{intermediate_table} (
-  LIKE #{table} INCLUDING ALL
-);
+CREATE TABLE #{intermediate_table} (LIKE #{table} INCLUDING ALL);
       SQL
 
       sql_format = SQL_FORMAT[period.to_sym]
       queries << <<-SQL
 CREATE FUNCTION #{trigger_name}()
-RETURNS trigger AS $$
-BEGIN
-  EXECUTE 'INSERT INTO public.#{table}_' || to_char(NEW.#{column}, '#{sql_format}') || ' VALUES ($1.*)' USING NEW;
-  RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
+    RETURNS trigger AS $$
+    BEGIN
+        EXECUTE 'INSERT INTO #{table}_' || to_char(NEW.#{column}, '#{sql_format}') || ' VALUES ($1.*)' USING NEW;
+        RETURN NULL;
+    END;
+    $$ LANGUAGE plpgsql;
       SQL
 
       queries << <<-SQL
 CREATE TRIGGER #{trigger_name}
-BEFORE INSERT ON #{intermediate_table}
-FOR EACH ROW EXECUTE PROCEDURE #{trigger_name}();
+    BEFORE INSERT ON #{intermediate_table}
+    FOR EACH ROW EXECUTE PROCEDURE #{trigger_name}();
       SQL
 
       run_queries(queries)
@@ -129,15 +129,13 @@ FOR EACH ROW EXECUTE PROCEDURE #{trigger_name}();
         partition_name = "#{original_table}_#{day.strftime(name_format)}"
         next if table_exists?(partition_name)
 
-        date_format = "%Y-%m-%d"
-
         queries << <<-SQL
-CREATE TABLE #{partition_name} (
-  CHECK (#{field} >= '#{day.strftime(date_format)}'::date AND #{field} < '#{(day + inc).strftime(date_format)}'::date)
-) INHERITS (#{table});
+CREATE TABLE #{partition_name}
+    (CHECK (#{field} >= #{sql_date(day)} AND #{field} < #{sql_date(day + inc)}))
+    INHERITS (#{table});
         SQL
 
-        queries << "ALTER TABLE #{partition_name} ADD PRIMARY KEY (#{primary_key});"
+        queries << "ALTER TABLE #{partition_name} ADD PRIMARY KEY (#{primary_key});" if primary_key
 
         index_defs.each do |index_def|
           queries << index_def.sub(" ON #{original_table} USING ", " ON #{partition_name} USING ").sub(/ INDEX .+ ON /, " INDEX ON ") + ";"
@@ -165,34 +163,44 @@ CREATE TABLE #{partition_name} (
 
       period, field, name_format, inc, today = settings_from_table(table, dest_table)
 
-      date_format = "%Y-%m-%d"
-      existing_tables = self.existing_tables(like: "#{table}_%").select { |t| /#{Regexp.escape("#{table}_")}(\d{4,6})/.match(t) }
+      existing_tables = self.existing_tables(like: "#{table}_%").select { |t| /#{Regexp.escape("#{table}_")}(\d{4,6})/.match(t) }.sort
       starting_time = DateTime.strptime(existing_tables.first.last(8), name_format)
       ending_time = DateTime.strptime(existing_tables.last.last(8), name_format) + inc
 
       primary_key = self.primary_key(table)
       max_source_id = max_id(source_table, primary_key)
-      max_dest_id = max_id(dest_table, primary_key)
+      max_dest_id =
+        if options[:swapped]
+          max_id(dest_table, primary_key, below: max_source_id)
+        else
+          max_id(dest_table, primary_key)
+        end
+
+      if max_dest_id == 0 && !options[:swapped]
+        min_source_id = min_id(source_table, primary_key, field, starting_time)
+        max_dest_id = min_source_id - 1 if min_source_id
+      end
 
-      starting_id = max_dest_id + 1
+      starting_id = max_dest_id
       fields = columns(source_table).join(", ")
       batch_size = options[:batch_size]
 
-      log "Overview"
-      log "#{source_table} max #{primary_key}: #{max_source_id}"
-      log "#{dest_table} max #{primary_key}: #{max_dest_id}"
-      log "time period: #{starting_time.to_date} -> #{ending_time.to_date}"
-      log
-
-      log "Batches"
-      while starting_id <= max_source_id
-        log "#{starting_id}..#{[starting_id + batch_size - 1, max_source_id].min}"
+      i = 1
+      batch_count = ((max_source_id - starting_id) / batch_size.to_f).ceil
+      while starting_id < max_source_id
+        query = <<-SQL
+/* #{i} of #{batch_count} */
+INSERT INTO #{dest_table} (#{fields})
+    SELECT #{fields} FROM #{source_table}
+    WHERE #{primary_key} > #{starting_id} AND #{primary_key} <= #{starting_id + batch_size} AND #{field} >= #{sql_date(starting_time)} AND #{field} < #{sql_date(ending_time)}
+        SQL
 
-        query = "INSERT INTO #{dest_table} (#{fields}) SELECT #{fields} FROM #{source_table} WHERE #{primary_key} >= #{starting_id} AND #{primary_key} < #{starting_id + batch_size} AND #{field} >= '#{starting_time.strftime(date_format)}'::date AND #{field} < '#{ending_time.strftime(date_format)}'::date"
-        log query if options[:debug]
+        log_sql(query)
+        log_sql
         execute(query)
 
         starting_id += batch_size
+        i += 1
 
         if options[:sleep] && starting_id <= max_source_id
           sleep(options[:sleep])
@@ -214,6 +222,11 @@ CREATE TABLE #{partition_name} (
         "ALTER TABLE #{table} RENAME TO #{retired_table};",
         "ALTER TABLE #{intermediate_table} RENAME TO #{table};"
       ]
+
+      self.sequences(table).each do |sequence|
+        queries << "ALTER SEQUENCE #{sequence["sequence_name"]} OWNED BY #{table}.#{sequence["related_column"]};"
+      end
+
       run_queries(queries)
     end
 
@@ -231,6 +244,11 @@ CREATE TABLE #{partition_name} (
         "ALTER TABLE #{table} RENAME TO #{intermediate_table};",
         "ALTER TABLE #{retired_table} RENAME TO #{table};"
       ]
+
+      self.sequences(table).each do |sequence|
+        queries << "ALTER SEQUENCE #{sequence["sequence_name"]} OWNED BY #{table}.#{sequence["related_column"]};"
+      end
+
       run_queries(queries)
     end
 
@@ -242,8 +260,8 @@ CREATE TABLE #{partition_name} (
         o.boolean "--swapped"
         o.boolean "--debug"
         o.float "--sleep"
-        o.integer "--future", default: 3
-        o.integer "--past", default: 3
+        o.integer "--future", default: 0
+        o.integer "--past", default: 0
         o.integer "--batch-size", default: 10000
         o.boolean "--dry-run", default: false
         o.on "-v", "--version", "print the version" do
@@ -343,12 +361,37 @@ CREATE TABLE #{partition_name} (
       row && row["attname"]
     end
 
-    def max_id(table, primary_key)
-      execute("SELECT MAX(#{primary_key}) FROM #{table}")[0]["max"].to_i
+    def max_id(table, primary_key, below: nil)
+      query = "SELECT MAX(#{primary_key}) FROM #{table}"
+      query << " WHERE #{primary_key} <= #{below}" if below
+      execute(query)[0]["max"].to_i
+    end
+
+    def min_id(table, primary_key, column, starting_time)
+      query = "SELECT MIN(#{primary_key}) FROM #{table} WHERE #{column} >= #{sql_date(starting_time)}"
+      execute(query)[0]["min"].to_i
     end
 
     def has_trigger?(trigger_name, table)
-      execute("SELECT 1 FROM pg_trigger WHERE tgname = $1 AND tgrelid = $2::regclass", [trigger_name, "public.#{table}"]).any?
+      execute("SELECT 1 FROM pg_trigger WHERE tgname = $1 AND tgrelid = $2::regclass", [trigger_name, table]).any?
+    end
+
+    # http://www.dbforums.com/showthread.php?1667561-How-to-list-sequences-and-the-columns-by-SQL
+    def sequences(table)
+      query = <<-SQL
+        SELECT
+          a.attname as related_column,
+          s.relname as sequence_name
+        FROM pg_class s
+          JOIN pg_depend d ON d.objid = s.oid
+          JOIN pg_class t ON d.objid = s.oid AND d.refobjid = t.oid
+          JOIN pg_attribute a ON (d.refobjid, d.refobjsubid) = (a.attrelid, a.attnum)
+          JOIN pg_namespace n ON n.oid = s.relnamespace
+        WHERE s.relkind = 'S'
+          AND n.nspname = 'public'
+          AND t.relname = $1
+      SQL
+      execute(query, [table])
     end
 
     # helpers
@@ -365,6 +408,10 @@ CREATE TABLE #{partition_name} (
       "#{table}_retired"
     end
 
+    def sql_date(time)
+      "'#{time.strftime("%Y-%m-%d")}'::date"
+    end
+
     def settings_from_table(original_table, table)
       trigger_name = self.trigger_name(original_table)
       function_def = execute("select pg_get_functiondef(oid) from pg_proc where proname = $1", [trigger_name])[0]["pg_get_functiondef"]

data/lib/pgslice/version.rb

@@ -1,3 +1,3 @@
 module PgSlice
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgslice
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Andrew Kane