postshift 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7af1b26becc44790eac020ebecc177aab32d7898
-  data.tar.gz: 7a2c30a03280b549744a963839a1f9faf0f0ca0b
+  metadata.gz: e91aea35f8015bd2dd96fdc881a749296153c051
+  data.tar.gz: 035a9e661aea4d78da25d6cc93febc05278fd766
 SHA512:
-  metadata.gz: 3dff95ff03d7f68c7c32255a671d7dbc96da14f39692b19c504acdbe049121586b24e7906c37a34510df549905a7894115a568e059b5d1f5fdec8c5ef07a3f7c
-  data.tar.gz: a7d61ba62cbe47d2486bb946a6621729e06f8b0ca0722652fee4882b13ee651a83fc16f62106575829e318ea046d275400d406cdfb6f9c0fb6e6639bdd06d20b
+  metadata.gz: 1f7b617a18904e274f57cbb73a75d2da9ccf48a225dd242729b8ce44caf278fd46aaf076864a139c4f14817f071c36c9b5faa73309541c9a299e026ff51ba996
+  data.tar.gz: 293edceeacae0d92e9b6e9d26b41807ffcb750cc48f78b18909dca5aa971895d37defc4a6843f0f525ad17c1029736f1b715f80e0c2ae9da91d8f44f8e082484

data/README.md

@@ -67,6 +67,25 @@ end
 For more information on compression types:
 <http://docs.aws.amazon.com/redshift/latest/dg/c_Compression_encodings.html>
 
+## Schema Dump
+
+Several Rake tasks exist to help with performing a schema dump of an existing Redshift database.  These utilize the [admin scripts provided by Amazon](https://github.com/awslabs/amazon-redshift-utils/tree/master/src/AdminViews).  All actions are performed using `ActiveRecord::Base.connection` by default.
+
+These views **must** exist within the Redshift database under the **admin** schema prior to any _dump_ or _restore_ actions.  To generate, simple run:
+
+```
+rake postshift:schema:prepare
+```
+
+Once these views are in place, the _dump_ and _restore_ will be available and functional.
+
+
+```
+rake postshift:schema:dump
+rake postshift:schema:restore
+```
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/postshift.rb

@@ -1,5 +1,18 @@
 require 'postshift/version'
+require 'postshift/railtie' if defined?(Rails)
+require 'postshift/schema'
+require 'active_support'
+require 'active_support/core_ext/module/attribute_accessors_per_thread'
 require 'active_record/connection_adapters/redshift_adapter'
 
 module Postshift
+  thread_mattr_accessor :adapter
+
+  def self.root
+    File.dirname __dir__
+  end
+
+  def self.connection
+    adapter.respond_to?(:raw_connection) ? adapter.raw_connection : nil
+  end
 end

data/lib/postshift/railtie.rb

@@ -0,0 +1,5 @@
+class Postshift::Railtie < Rails::Railtie
+  rake_tasks do
+    load 'tasks/schema.rake'
+  end
+end

data/lib/postshift/schema.rb

@@ -0,0 +1,100 @@
+module Postshift
+  module Schema
+    FILENAME = 'postshift_schema.sql'.freeze
+    ADMIN_UTILITIES = %w(v_generate_tbl_ddl v_generate_view_ddl).freeze
+
+    def self.ensure_admin_schema
+      Postshift.connection.exec('CREATE SCHEMA IF NOT EXISTS admin')
+    end
+
+    def self.create_admin_utilities!
+      ensure_admin_schema
+      ADMIN_UTILITIES.each do |table_name|
+        Postshift.connection.exec(generate_ddl_sql(table_name))
+      end
+    end
+
+    def self.remove_admin_utilities!
+      ensure_admin_schema
+      ADMIN_UTILITIES.each do |table_name|
+        Postshift.connection.exec("DROP VIEW IF EXISTS admin.#{table_name}")
+      end
+    end
+
+    def self.admin_uilities_exists?
+      ensure_admin_schema
+      ADMIN_UTILITIES.each do |table_name|
+        Postshift.connection.exec("SELECT 'admin.#{table_name}'::regclass;")
+      end
+      true
+    rescue PG::UndefinedTable
+      false
+    end
+
+    def self.generate_ddl_sql(table_name)
+      path = File.join(Postshift.root, 'lib', 'tasks', "#{table_name}.sql")
+      File.open(path).read
+    end
+
+    def self.dump
+      File.open(output_location, 'w+') do |file|
+        ddl_results(tbl_ddl_sql).each_row do |row|
+          file.puts(row)
+        end
+        ddl_results(view_ddl_sql).each_row do |row|
+          file.puts(row)
+        end
+      end
+    end
+
+    def self.dump_sql
+      if File.exist?(output_location)
+        File.read(output_location)
+      else
+        puts 'Postshift Schema Dump file does not exist. Run task postshift:schema:dump'
+        false
+      end
+    end
+
+    def self.restore
+      sql = dump_sql
+      Postshift.connection.exec(sql) if sql.present?
+    end
+
+    def self.ddl_results(ddl_sql)
+      Postshift.connection.exec(ddl_sql, schemas)
+    end
+
+    def self.output_location
+      if defined?(Rails)
+        File.join(Rails.root, 'db', FILENAME)
+      else
+        base_path = File.join(Postshift.root, 'tmp')
+        Dir.mkdir(base_path) unless Dir.exist?(base_path)
+        File.join(base_path, FILENAME)
+      end
+    end
+
+    def self.schemas
+      %w(public)
+    end
+
+    def self.tbl_ddl_sql
+      <<-SQL
+        SELECT  ddl
+        FROM    admin.v_generate_tbl_ddl
+        WHERE   schemaname IN ($1)
+        ORDER BY tablename ASC, seq ASC
+      SQL
+    end
+
+    def self.view_ddl_sql
+      <<-SQL
+        SELECT  ddl
+        FROM    admin.v_generate_view_ddl
+        WHERE   schemaname IN ($1)
+        ORDER BY viewname ASC
+      SQL
+    end
+  end
+end

data/lib/postshift/version.rb

@@ -1,3 +1,3 @@
 module Postshift
-  VERSION = '0.1.2'
+  VERSION = '0.2.0'
 end

data/lib/tasks/schema.rake

@@ -0,0 +1,28 @@
+namespace :postshift do
+  namespace :schema do
+    task connect: :environment do
+      Postshift.adapter = ActiveRecord::Base.connection
+    end
+
+    desc 'Prepare Redshift instance for Schema Exports'
+    task prepare: :connect do
+      puts '*** Creating Admin Utilities ***'
+      Postshift::Schema.create_admin_utilities!
+      puts '*** Finished Creating Admin Utilities ***'
+    end
+
+    desc 'Creates a db/postshift_schema.sql file for Redshift databases'
+    task dump: :connect do
+      puts '*** Starting Postshift Schema Dump ***'
+      Postshift::Schema.dump
+      puts '*** Completed Postshift Schema Dump ***'
+    end
+
+    desc 'Restores db/postshift_schema.sql file to Redshift database'
+    task restore: :connect do
+      puts '*** Starting Postshift Schema Restore ***'
+      Postshift::Schema.restore
+      puts '*** Completed Postshift Schema Restore ***'
+    end
+  end
+end

data/lib/tasks/v_generate_tbl_ddl.sql

@@ -0,0 +1,254 @@
+--DROP VIEW admin.v_generate_tbl_ddl;
+/**********************************************************************************************
+Purpose: View to get the DDL for a table.  This will contain the distkey, sortkey, constraints,
+         not null, defaults, etc.
+
+Notes:   Default view ordering causes foreign keys to be created at the end.
+         This is needed due to dependencies of the foreign key constraint and the tables it
+         links.  Due to this one should not manually order the output if you are expecting to
+         be able to replay the SQL directly from the VIEW query result. It is still possible to
+         order if you filter out the FOREIGN KEYS and then apply them later.
+
+         The following filters are useful:
+           where ddl not like 'ALTER TABLE %'  -- do not return FOREIGN KEY CONSTRAINTS
+           where ddl like 'ALTER TABLE %'      -- only get FOREIGN KEY CONSTRAINTS
+           where tablename in ('t1', 't2')     -- only get DDL for specific tables
+           where schemaname in ('s1', 's2')    -- only get DDL for specific schemas
+
+         So for example if you want to order DDL on tablename and only want the tables 't1', 't2'
+         and 't4' you can do so by using a query like:
+           select ddl from (
+             (
+               select
+                 *
+               from admin.v_generate_tbl_ddl
+               where ddl not like 'ALTER TABLE %'
+               order by tablename
+             )
+             UNION ALL
+             (
+               select
+                 *
+               from admin.v_generate_tbl_ddl
+               where ddl like 'ALTER TABLE %'
+               order by tablename
+             )
+           ) where tablename in ('t1', 't2', 't4');
+
+History:
+2014-02-10 jjschmit Created
+2015-05-18 ericfe Added support for Interleaved sortkey
+2015-10-31 ericfe Added cast tp increase size of returning constraint name
+2016-05-24 chriz-bigdata Added support for BACKUP NO tables
+2017-05-03 pvbouwel Change table & schemaname of Foreign key constraints to allow for filters
+**********************************************************************************************/
+CREATE OR REPLACE VIEW admin.v_generate_tbl_ddl
+AS
+SELECT
+ REGEXP_REPLACE (schemaname, '^zzzzzzzz', '') AS schemaname
+ ,REGEXP_REPLACE (tablename, '^zzzzzzzz', '') AS tablename
+ ,seq
+ ,ddl
+FROM
+ (
+ SELECT
+  schemaname
+  ,tablename
+  ,seq
+  ,ddl
+ FROM
+  (
+  --DROP TABLE
+  SELECT
+   n.nspname AS schemaname
+   ,c.relname AS tablename
+   ,0 AS seq
+   ,'--DROP TABLE "' + n.nspname + '"."' + c.relname + '";' AS ddl
+  FROM pg_namespace AS n
+  INNER JOIN pg_class AS c ON n.oid = c.relnamespace
+  WHERE c.relkind = 'r'
+  --CREATE TABLE
+  UNION SELECT
+   n.nspname AS schemaname
+   ,c.relname AS tablename
+   ,2 AS seq
+   ,'CREATE TABLE IF NOT EXISTS "' + n.nspname + '"."' + c.relname + '"' AS ddl
+  FROM pg_namespace AS n
+  INNER JOIN pg_class AS c ON n.oid = c.relnamespace
+  WHERE c.relkind = 'r'
+  --OPEN PAREN COLUMN LIST
+  UNION SELECT n.nspname AS schemaname, c.relname AS tablename, 5 AS seq, '(' AS ddl
+  FROM pg_namespace AS n
+  INNER JOIN pg_class AS c ON n.oid = c.relnamespace
+  WHERE c.relkind = 'r'
+  --COLUMN LIST
+  UNION SELECT
+   schemaname
+   ,tablename
+   ,seq
+   ,'\t' + col_delim + col_name + ' ' + col_datatype + ' ' + col_nullable + ' ' + col_default + ' ' + col_encoding AS ddl
+  FROM
+   (
+   SELECT
+    n.nspname AS schemaname
+    ,c.relname AS tablename
+    ,100000000 + a.attnum AS seq
+    ,CASE WHEN a.attnum > 1 THEN ',' ELSE '' END AS col_delim
+    ,'"' + a.attname + '"' AS col_name
+    ,CASE WHEN STRPOS(UPPER(format_type(a.atttypid, a.atttypmod)), 'CHARACTER VARYING') > 0
+      THEN REPLACE(UPPER(format_type(a.atttypid, a.atttypmod)), 'CHARACTER VARYING', 'VARCHAR')
+     WHEN STRPOS(UPPER(format_type(a.atttypid, a.atttypmod)), 'CHARACTER') > 0
+      THEN REPLACE(UPPER(format_type(a.atttypid, a.atttypmod)), 'CHARACTER', 'CHAR')
+     ELSE UPPER(format_type(a.atttypid, a.atttypmod))
+     END AS col_datatype
+    ,CASE WHEN format_encoding((a.attencodingtype)::integer) = 'none'
+     THEN ''
+     ELSE 'ENCODE ' + format_encoding((a.attencodingtype)::integer)
+     END AS col_encoding
+    ,CASE WHEN a.atthasdef IS TRUE THEN 'DEFAULT ' + adef.adsrc ELSE '' END AS col_default
+    ,CASE WHEN a.attnotnull IS TRUE THEN 'NOT NULL' ELSE '' END AS col_nullable
+   FROM pg_namespace AS n
+   INNER JOIN pg_class AS c ON n.oid = c.relnamespace
+   INNER JOIN pg_attribute AS a ON c.oid = a.attrelid
+   LEFT OUTER JOIN pg_attrdef AS adef ON a.attrelid = adef.adrelid AND a.attnum = adef.adnum
+   WHERE c.relkind = 'r'
+     AND a.attnum > 0
+   ORDER BY a.attnum
+   )
+  --CONSTRAINT LIST
+  UNION (SELECT
+   n.nspname AS schemaname
+   ,c.relname AS tablename
+   ,200000000 + CAST(con.oid AS INT) AS seq
+   ,'\t,' + pg_get_constraintdef(con.oid) AS ddl
+  FROM pg_constraint AS con
+  INNER JOIN pg_class AS c ON c.relnamespace = con.connamespace AND c.oid = con.conrelid
+  INNER JOIN pg_namespace AS n ON n.oid = c.relnamespace
+  WHERE c.relkind = 'r' AND pg_get_constraintdef(con.oid) NOT LIKE 'FOREIGN KEY%'
+  ORDER BY seq)
+  --CLOSE PAREN COLUMN LIST
+  UNION SELECT n.nspname AS schemaname, c.relname AS tablename, 299999999 AS seq, ')' AS ddl
+  FROM pg_namespace AS n
+  INNER JOIN pg_class AS c ON n.oid = c.relnamespace
+  WHERE c.relkind = 'r'
+  --BACKUP
+  UNION SELECT
+  n.nspname AS schemaname
+   ,c.relname AS tablename
+   ,300000000 AS seq
+   ,'BACKUP NO' as ddl
+FROM pg_namespace AS n
+  INNER JOIN pg_class AS c ON n.oid = c.relnamespace
+  INNER JOIN (SELECT
+    SPLIT_PART(key,'_',5) id
+    FROM pg_conf
+    WHERE key LIKE 'pg_class_backup_%'
+    AND SPLIT_PART(key,'_',4) = (SELECT
+      oid
+      FROM pg_database
+      WHERE datname = current_database())) t ON t.id=c.oid
+  WHERE c.relkind = 'r'
+  --BACKUP WARNING
+  UNION SELECT
+  n.nspname AS schemaname
+   ,c.relname AS tablename
+   ,1 AS seq
+   ,'--WARNING: This DDL inherited the BACKUP NO property from the source table' as ddl
+FROM pg_namespace AS n
+  INNER JOIN pg_class AS c ON n.oid = c.relnamespace
+  INNER JOIN (SELECT
+    SPLIT_PART(key,'_',5) id
+    FROM pg_conf
+    WHERE key LIKE 'pg_class_backup_%'
+    AND SPLIT_PART(key,'_',4) = (SELECT
+      oid
+      FROM pg_database
+      WHERE datname = current_database())) t ON t.id=c.oid
+  WHERE c.relkind = 'r'
+  --DISTSTYLE
+  UNION SELECT
+   n.nspname AS schemaname
+   ,c.relname AS tablename
+   ,300000001 AS seq
+   ,CASE WHEN c.reldiststyle = 0 THEN 'DISTSTYLE EVEN'
+    WHEN c.reldiststyle = 1 THEN 'DISTSTYLE KEY'
+    WHEN c.reldiststyle = 8 THEN 'DISTSTYLE ALL'
+    ELSE '<<Error - UNKNOWN DISTSTYLE>>'
+    END AS ddl
+  FROM pg_namespace AS n
+  INNER JOIN pg_class AS c ON n.oid = c.relnamespace
+  WHERE c.relkind = 'r'
+  --DISTKEY COLUMNS
+  UNION SELECT
+   n.nspname AS schemaname
+   ,c.relname AS tablename
+   ,400000000 + a.attnum AS seq
+   ,'DISTKEY ("' + a.attname + '")' AS ddl
+  FROM pg_namespace AS n
+  INNER JOIN pg_class AS c ON n.oid = c.relnamespace
+  INNER JOIN pg_attribute AS a ON c.oid = a.attrelid
+  WHERE c.relkind = 'r'
+    AND a.attisdistkey IS TRUE
+    AND a.attnum > 0
+  --SORTKEY COLUMNS
+  UNION select schemaname, tablename, seq,
+       case when min_sort <0 then 'INTERLEAVED SORTKEY (' else 'SORTKEY (' end as ddl
+from (SELECT
+   n.nspname AS schemaname
+   ,c.relname AS tablename
+   ,499999999 AS seq
+   ,min(attsortkeyord) min_sort FROM pg_namespace AS n
+  INNER JOIN  pg_class AS c ON n.oid = c.relnamespace
+  INNER JOIN pg_attribute AS a ON c.oid = a.attrelid
+  WHERE c.relkind = 'r'
+  AND abs(a.attsortkeyord) > 0
+  AND a.attnum > 0
+  group by 1,2,3 )
+  UNION (SELECT
+   n.nspname AS schemaname
+   ,c.relname AS tablename
+   ,500000000 + abs(a.attsortkeyord) AS seq
+   ,CASE WHEN abs(a.attsortkeyord) = 1
+    THEN '\t"' + a.attname + '"'
+    ELSE '\t, "' + a.attname + '"'
+    END AS ddl
+  FROM  pg_namespace AS n
+  INNER JOIN pg_class AS c ON n.oid = c.relnamespace
+  INNER JOIN pg_attribute AS a ON c.oid = a.attrelid
+  WHERE c.relkind = 'r'
+    AND abs(a.attsortkeyord) > 0
+    AND a.attnum > 0
+  ORDER BY abs(a.attsortkeyord))
+  UNION SELECT
+   n.nspname AS schemaname
+   ,c.relname AS tablename
+   ,599999999 AS seq
+   ,'\t)' AS ddl
+  FROM pg_namespace AS n
+  INNER JOIN  pg_class AS c ON n.oid = c.relnamespace
+  INNER JOIN  pg_attribute AS a ON c.oid = a.attrelid
+  WHERE c.relkind = 'r'
+    AND abs(a.attsortkeyord) > 0
+    AND a.attnum > 0
+  --END SEMICOLON
+  UNION SELECT n.nspname AS schemaname, c.relname AS tablename, 600000000 AS seq, ';' AS ddl
+  FROM  pg_namespace AS n
+  INNER JOIN pg_class AS c ON n.oid = c.relnamespace
+  WHERE c.relkind = 'r' )
+  UNION (
+    SELECT 'zzzzzzzz' || n.nspname AS schemaname,
+       'zzzzzzzz' || c.relname AS tablename,
+       700000000 + CAST(con.oid AS INT) AS seq,
+       'ALTER TABLE ' + n.nspname + '.' + c.relname + ' ADD ' + pg_get_constraintdef(con.oid)::VARCHAR(1024) + ';' AS ddl
+    FROM pg_constraint AS con
+      INNER JOIN pg_class AS c
+             ON c.relnamespace = con.connamespace
+             AND c.oid = con.conrelid
+      INNER JOIN pg_namespace AS n ON n.oid = c.relnamespace
+    WHERE c.relkind = 'r'
+    AND con.contype = 'f'
+    ORDER BY seq
+  )
+ ORDER BY schemaname, tablename, seq
+ )
+;

data/lib/tasks/v_generate_view_ddl.sql

@@ -0,0 +1,19 @@
+--DROP VIEW admin.v_generate_view_ddl;
+/**********************************************************************************************
+Purpose: View to get the DDL for a view.  
+History:
+2014-02-10 jjschmit Created
+**********************************************************************************************/
+CREATE OR REPLACE VIEW admin.v_generate_view_ddl
+AS
+SELECT 
+	n.nspname AS schemaname
+	,c.relname AS viewname
+	,'--DROP VIEW ' + n.nspname + '.' + c.relname + ';\nCREATE OR REPLACE VIEW ' + n.nspname + '.' + c.relname + ' AS\n' + COALESCE(pg_get_viewdef(c.oid, TRUE), '') AS ddl
+FROM 
+	pg_catalog.pg_class AS c
+INNER JOIN
+	pg_catalog.pg_namespace AS n
+		ON c.relnamespace = n.oid
+WHERE relkind = 'v'
+;

data/postshift.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 1.14'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'factory_girl', '~> 4.8.0'
-  spec.add_development_dependency 'database_cleaner', '~> 1.6.0'
-  spec.add_development_dependency 'appraisal', '~> 2.2.0'
+  spec.add_development_dependency 'factory_girl', '~> 4.8'
+  spec.add_development_dependency 'database_cleaner', '~> 1.6'
+  spec.add_development_dependency 'appraisal', '~> 2.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postshift
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Dave Krupinski
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-29 00:00:00.000000000 Z
+date: 2017-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -106,42 +106,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.8.0
+        version: '4.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.8.0
+        version: '4.8'
 - !ruby/object:Gem::Dependency
   name: database_cleaner
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: '1.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: '2.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: '2.2'
 description: Thin ActiveRecord Redshift Adapter extension for existing PostgreSQL
   adapter.
 email:
@@ -176,7 +176,12 @@ files:
 - lib/active_record/connection_adapters/redshift/type_metadata.rb
 - lib/active_record/connection_adapters/redshift_adapter.rb
 - lib/postshift.rb
+- lib/postshift/railtie.rb
+- lib/postshift/schema.rb
 - lib/postshift/version.rb
+- lib/tasks/schema.rake
+- lib/tasks/v_generate_tbl_ddl.sql
+- lib/tasks/v_generate_view_ddl.sql
 - postshift.gemspec
 homepage: https://github.com/ValiMail/postshift
 licenses: