postgres_to_redshift 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b9aca81574e39acdc400fcfd77f454cb9c93f682
-  data.tar.gz: f4a8430406d25028ff1ac9b305837d34d0463464
+  metadata.gz: 085f85fd5e501b248eeab36aa540449977881d45
+  data.tar.gz: 21a8909abb27984a06b7d23b25f0ebda47647282
 SHA512:
-  metadata.gz: 8136aaffdd83cb676290393235f3cb1e99cf257c8c3dc193c0b234feec09b3f3cfedb29a8e6ad7f9f55641a2e1ef02c624f6d86073add5fec9ee5e0a2eacdde4
-  data.tar.gz: c0b9b49f1ba41e26fd2bb87328c5f672648a98ddd432335d8a899f6dbd722b1c32b6d6cb85b4de19630e42690eaf835957a6e801001d8a46972f917f0678482e
+  metadata.gz: 48dad16338a06b1deb436c1bcf3448b461444ac94697802a3e4f7ec2877ce65f6b2ade48cfc3f70cd7062e55996c8a0ab28c8553d9340d18ff0fe3c785a0b3d6
+  data.tar.gz: 5f08cae1688d81e6034083394072a06774915d9e2d1511f8cd56c27572226be3ef952eb5f0b4ebdf871946cdcdba2855cc5c4c41f880e021d66c8127a6cf5224

data/.gitignore

@@ -14,3 +14,5 @@
 *.a
 mkmf.log
 *swp
+.env
+.idea

data/.rspec

@@ -1,2 +1,3 @@
 --color
 --require spec_helper
+--tag ~type:feature

data/README.md

@@ -31,7 +31,7 @@ export S3_DATABASE_EXPORT_ID='yourid'
 export S3_DATABASE_EXPORT_KEY='yourkey'
 export S3_DATABASE_EXPORT_BUCKET='some-bucket-to-use'
 
-postgres_to_redshift $MY_SOURCE_DATABASE
+postgres_to_redshift
 ```
 
 ## Contributing

data/Rakefile

@@ -1,10 +1,17 @@
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
 
 begin
   require 'rspec/core/rake_task'
-  RSpec::Core::RakeTask.new(:spec)
+  namespace :spec do
+    RSpec::Core::RakeTask.new(:units) do |t|
+      t.rspec_opts = '--tag ~type:feature'
+    end
+    RSpec::Core::RakeTask.new(:features) do |t|
+      t.rspec_opts = '--tag type:feature'
+    end
+  end
 rescue LoadError
 end
 
 task(:default).clear
-task :default => :spec
+task :default => 'spec:units'

data/bin/configure_aws.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+S3_DATABASE_EXPORT_ID=""
+S3_DATABASE_EXPORT_KEY=""
+
+source ${DIR}/../.env
+
+if [ ${S3_DATABASE_EXPORT_ID} = "" ]; then
+    exit 1
+fi
+
+if [ ${S3_DATABASE_EXPORT_KEY} = "" ]; then
+    exit 1
+fi
+
+aws configure set aws_access_key_id ${S3_DATABASE_EXPORT_ID} --profile p2r
+aws configure set aws_secret_access_key ${S3_DATABASE_EXPORT_KEY} --profile p2r
+aws configure set default.region us-east-1 --profile p2r

data/bin/create_stack.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source ${DIR}/../.env
+
+aws cloudformation describe-stacks --profile p2r --stack-name p2r
+EXISTS=$?
+
+if (( EXISTS == 0 )); then
+    echo "Stack already exists. Exiting."
+exit 1
+fi
+
+aws cloudformation create-stack --profile p2r --stack-name p2r --template-body file:///${DIR}/../config/cloud-formation-local-postgres.json --on-failure DELETE
+if [ $? -ne 0 ]; then
+    echo "Error creating stack. Exiting."
+fi
+
+echo "Waiting for stack to finish creating."
+aws cloudformation wait stack-create-complete --profile p2r --stack-name p2r

data/bin/delete_stack.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source ${DIR}/../.env
+
+aws cloudformation describe-stacks --profile p2r --stack-name p2r
+if [ $? -ne 0 ]; then
+    echo "Stack does not exist. Exiting."
+exit 1
+fi
+
+aws cloudformation delete-stack --profile p2r --stack-name p2r
+if [ $? -ne 0 ]; then
+    echo "Error deleting stack. Exiting."
+fi
+
+echo "Waiting for stack to finish deleting."
+aws cloudformation wait stack-delete-complete --profile p2r --stack-name p2r
+if [ $? -ne 0 ]; then
+    echo "Error waiting for stack to delete. Exiting."
+fi
+
+echo "Stack deleted."

data/bin/stack_env.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+aws cloudformation describe-stacks --profile p2r --stack-name p2r > /dev/null
+if [ $? -ne 0 ]; then
+    echo "Stack does not exist. Exiting."
+exit 1
+fi
+
+export S3_DATABASE_EXPORT_BUCKET=$(aws cloudformation describe-stack-resources --profile p2r --stack-name p2r --logical-resource-id s3Bucket | jq -r ".StackResources[0].PhysicalResourceId")
+P2R_REDSHIFT_ID=$(aws cloudformation describe-stack-resources --profile p2r --stack-name p2r --logical-resource-id Redshift | jq -r ".StackResources[0].PhysicalResourceId")
+P2R_REDSHIFT_ADDRESS=$(aws redshift describe-clusters --profile p2r --cluster-identifier ${P2R_REDSHIFT_ID} | jq -r ".Clusters[0].Endpoint.Address")
+export POSTGRES_TO_REDSHIFT_TARGET_URI="postgres://test:Testtesttest1@${P2R_REDSHIFT_ADDRESS}:5439/test"
+
+echo "S3_DATABASE_EXPORT_BUCKET=${P2R_S3_BUCKET}"
+echo "POSTGRES_TO_REDSHIFT_TARGET_URI=${POSTGRES_TO_REDSHIFT_TARGET_URI}"

data/bin/update_stack.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source ${DIR}/../.env
+
+aws cloudformation describe-stacks --profile p2r --stack-name p2r
+if [ $? -ne 0 ]; then
+    echo "Stack does not exist. Exiting."
+exit 1
+fi
+
+aws cloudformation update-stack --profile p2r --stack-name p2r --template-body file:///${DIR}/../config/cloud-formation-local-postgres.json --on-failure DELETE
+if [ $? -ne 0 ]; then
+    echo "Error updating stack. Exiting."
+fi
+
+echo "Waiting for stack to finish updating."
+aws cloudformation wait stack-update-complete --profile p2r --stack-name p2r

data/config/cloud-formation-local-postgres.json

@@ -0,0 +1,39 @@
+{
+  "AWSTemplateFormatVersion": "2010-09-09",
+  "Metadata": {
+  },
+  "Resources": {
+    "redshiftSecurityGroup" : {
+      "Type": "AWS::Redshift::ClusterSecurityGroup",
+      "Properties": {
+        "Description" : "Security group to determine where connections to the Amazon Redshift cluster can come from"
+      }
+    },
+    "redshiftSecurityGroupIngress" : {
+      "Type": "AWS::Redshift::ClusterSecurityGroupIngress",
+      "Properties": {
+        "ClusterSecurityGroupName" : {"Ref":"redshiftSecurityGroup"},
+        "CIDRIP" : "0.0.0.0/0"
+      }
+    },
+    "Redshift": {
+      "Type": "AWS::Redshift::Cluster",
+      "Properties": {
+        "ClusterType": "single-node",
+        "ClusterSecurityGroups": [{"Ref":"redshiftSecurityGroup"}],
+        "DBName": "test",
+        "MasterUsername": "test",
+        "MasterUserPassword": "Testtesttest1",
+        "NodeType": "dc1.large"
+      },
+      "Metadata": {
+      }
+    },
+    "s3Bucket": {
+      "Type": "AWS::S3::Bucket",
+      "Properties": {},
+      "Metadata": {
+      }
+    }
+  }
+}

data/config/cloud-formation-rds.json

@@ -0,0 +1,55 @@
+{
+  "AWSTemplateFormatVersion": "2010-09-09",
+  "Metadata": {
+  },
+  "Resources": {
+    "postgresSecurityGroup" : {
+       "Type" : "AWS::RDS::DBSecurityGroup",
+       "Properties" :
+       {
+          "DBSecurityGroupIngress" : [ {"CIDRIP": "0.0.0.0/0"} ],
+          "GroupDescription" : "Sup"
+       }
+    },
+    "postgres": {
+      "Type": "AWS::RDS::DBInstance",
+      "Properties": {
+        "AllowMajorVersionUpgrade": true,
+        "DBInstanceClass": "db.m3.medium",
+        "DBSecurityGroups": [{"Ref" :"postgresSecurityGroup"}],
+        "BackupRetentionPeriod": "0",
+        "DBName": "test",
+        "AllocatedStorage": "5",
+        "Engine": "postgres",
+        "EngineVersion": "9.5.2",
+        "MasterUsername": "test",
+        "MasterUserPassword": "testtesttest",
+        "StorageType": "standard",
+        "PubliclyAccessible": true
+      },
+      "Metadata": {
+      },
+      "DependsOn": [
+        "postgresSecurityGroup"
+      ]
+    },
+    "Redshift": {
+      "Type": "AWS::Redshift::Cluster",
+      "Properties": {
+        "ClusterType": "single-node",
+        "DBName": "test",
+        "MasterUsername": "test",
+        "MasterUserPassword": "Testtesttest1",
+        "NodeType": "dc1.large"
+      },
+      "Metadata": {
+      }
+    },
+    "s3Bucket": {
+      "Type": "AWS::S3::Bucket",
+      "Properties": {},
+      "Metadata": {
+      }
+    }
+  }
+}

data/lib/postgres_to_redshift.rb

@@ -1,9 +1,9 @@
 require "postgres_to_redshift/version"
 require 'pg'
 require 'uri'
-require 'aws-sdk'
+require 'aws-sdk-v1'
 require 'zlib'
-require 'stringio'
+require 'tempfile'
 require "postgres_to_redshift/table"
 require "postgres_to_redshift/column"
 
@@ -14,11 +14,15 @@ class PostgresToRedshift
 
   attr_reader :source_connection, :target_connection, :s3
 
+  KILOBYTE = 1024
+  MEGABYTE = KILOBYTE * 1024
+  GIGABYTE = MEGABYTE * 1024
+
   def self.update_tables
     update_tables = PostgresToRedshift.new
 
     update_tables.tables.each do |table|
-      target_connection.exec("CREATE TABLE IF NOT EXISTS public.#{table.target_table_name} (#{table.columns_for_create})")
+      target_connection.exec("CREATE TABLE IF NOT EXISTS public.#{target_connection.quote_ident(table.target_table_name)} (#{table.columns_for_create})")
 
       update_tables.copy_table(table)
 
@@ -81,26 +85,42 @@ class PostgresToRedshift
   end
 
   def copy_table(table)
-    buffer = StringIO.new
-    zip = Zlib::GzipWriter.new(buffer)
-
-    puts "Downloading #{table}"
-    copy_command = "COPY (SELECT #{table.columns_for_copy} FROM #{table.name}) TO STDOUT WITH DELIMITER '|'"
-
-    source_connection.copy_data(copy_command) do
-      while row = source_connection.get_copy_data
-        zip.write(row)
+    tmpfile = Tempfile.new("psql2rs")
+    zip = Zlib::GzipWriter.new(tmpfile)
+    chunksize = 5 * GIGABYTE # uncompressed
+    chunk = 1
+    bucket.objects.with_prefix("export/#{table.target_table_name}.psv.gz").delete_all
+    begin
+      puts "Downloading #{table}"
+      copy_command = "COPY (SELECT #{table.columns_for_copy} FROM #{table.name}) TO STDOUT WITH DELIMITER '|'"
+
+      source_connection.copy_data(copy_command) do
+        while row = source_connection.get_copy_data
+          zip.write(row)
+          if (zip.pos > chunksize)
+            zip.finish
+            tmpfile.rewind
+            upload_table(table, tmpfile, chunk)
+            chunk += 1
+            zip.close unless zip.closed?
+            tmpfile.unlink
+            tmpfile = Tempfile.new("psql2rs")
+            zip = Zlib::GzipWriter.new(tmpfile)
+          end
+        end
       end
+      zip.finish
+      tmpfile.rewind
+      upload_table(table, tmpfile, chunk)
+    ensure
+      zip.close unless zip.closed?
+      tmpfile.unlink
     end
-    zip.finish
-    buffer.rewind
-    upload_table(table, buffer)
   end
 
-  def upload_table(table, buffer)
-    puts "Uploading #{table.target_table_name}"
-    bucket.objects["export/#{table.target_table_name}.psv.gz"].delete
-    bucket.objects["export/#{table.target_table_name}.psv.gz"].write(buffer, acl: :authenticated_read)
+  def upload_table(table, buffer, chunk)
+    puts "Uploading #{table.target_table_name}.#{chunk}"
+    bucket.objects["export/#{table.target_table_name}.psv.gz.#{chunk}"].write(buffer, acl: :authenticated_read)
   end
 
   def import_table(table)
@@ -109,11 +129,11 @@ class PostgresToRedshift
 
     target_connection.exec("BEGIN;")
 
-    target_connection.exec("ALTER TABLE public.#{table.target_table_name} RENAME TO #{table.target_table_name}_updating")
+    target_connection.exec("ALTER TABLE public.#{target_connection.quote_ident(table.target_table_name)} RENAME TO #{table.target_table_name}_updating")
 
-    target_connection.exec("CREATE TABLE public.#{table.target_table_name} (#{table.columns_for_create})")
+    target_connection.exec("CREATE TABLE public.#{target_connection.quote_ident(table.target_table_name)} (#{table.columns_for_create})")
 
-    target_connection.exec("COPY public.#{table.target_table_name} FROM 's3://#{ENV['S3_DATABASE_EXPORT_BUCKET']}/export/#{table.target_table_name}.psv.gz' CREDENTIALS 'aws_access_key_id=#{ENV['S3_DATABASE_EXPORT_ID']};aws_secret_access_key=#{ENV['S3_DATABASE_EXPORT_KEY']}' GZIP TRUNCATECOLUMNS ESCAPE DELIMITER as '|';")
+    target_connection.exec("COPY public.#{target_connection.quote_ident(table.target_table_name)} FROM 's3://#{ENV['S3_DATABASE_EXPORT_BUCKET']}/export/#{table.target_table_name}.psv.gz' CREDENTIALS 'aws_access_key_id=#{ENV['S3_DATABASE_EXPORT_ID']};aws_secret_access_key=#{ENV['S3_DATABASE_EXPORT_KEY']}' GZIP TRUNCATECOLUMNS ESCAPE DELIMITER as '|';")
 
     target_connection.exec("COMMIT;")
   end

data/lib/postgres_to_redshift/version.rb

@@ -1,3 +1,3 @@
 class PostgresToRedshift
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/postgres_to_redshift.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ["arakoczy@gmail.com"]
   spec.summary       = %q{Load postgres databases into Amazon Redshift}
   spec.description   = %q{Load postgres databases into Amazon Redshift. It's designed to work on Heroku Scheduler, or other *nix/BSD hosts.}
-  spec.homepage      = "https://github.com/kitchensurfing/postgres_to_redshift"
+  spec.homepage      = "https://github.com/toothrot/postgres_to_redshift"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0")
@@ -21,5 +21,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_dependency "pg", "~> 0.17.0"
-  spec.add_dependency "aws-sdk", "~> 1.54"
+  spec.add_dependency "aws-sdk-v1", "~> 1.54"
 end

data/spec/features/small_table_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+RSpec.describe 'a small source database', type: :feature do
+  context 'with a simple column' do
+    before(:all) do
+      PostgresToRedshift::Test.test_connection.exec(<<-EOS
+        DROP TABLE IF EXISTS "p2r_integration";
+        CREATE TABLE IF NOT EXISTS "p2r_integration" ("id" SERIAL PRIMARY KEY, "title" text);
+        INSERT INTO "p2r_integration" ("title") VALUES ('Casablanca');
+      EOS
+      )
+    end
+    after(:all) do
+      PostgresToRedshift::Test.test_connection.exec(%q[DROP TABLE IF EXISTS "p2r_integration";])
+      PostgresToRedshift::Test.test_target_connection.exec(%q[DROP TABLE IF EXISTS "p2r_integration";])
+
+    end
+
+    it 'Copies all rows to target table' do
+      PostgresToRedshift.update_tables
+      result = PostgresToRedshift::Test.test_target_connection.exec(
+          'SELECT * FROM "p2r_integration";'
+      )
+      expect(result.num_tuples).to eq(1)
+      expect(result[0]).to eq('title' => 'Casablanca', 'id' => '1')
+    end
+  end
+end

data/spec/spec_prepare.rb

@@ -3,8 +3,28 @@ module PostgresToRedshift::Test
     PostgresToRedshift.source_uri
   end
 
+  def self.target_uri
+    PostgresToRedshift.target_uri
+  end
+
   def self.test_connection
-    @test_connection ||= PG::Connection.new(host: source_uri.host, port: source_uri.port, user: source_uri.user || ENV['USER'], password: source_uri.password, dbname: source_uri.path[1..-1])
+    @test_connection ||= PG::Connection.new(
+        host: source_uri.host,
+        port: source_uri.port,
+        user: source_uri.user || ENV['USER'],
+        password: source_uri.password,
+        dbname: source_uri.path[1..-1]
+    )
+  end
+
+  def self.test_target_connection
+    @test_connection ||= PG::Connection.new(
+        host: target_uri.host,
+        port: target_uri.port,
+        user: target_uri.user || ENV['USER'],
+        password: target_uri.password,
+        dbname: target_uri.path[1..-1]
+    )
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postgres_to_redshift
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Alex Rakoczy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-06 00:00:00.000000000 Z
+date: 2016-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -53,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.17.0
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-v1
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -71,7 +71,12 @@ description: Load postgres databases into Amazon Redshift. It's designed to work
 email:
 - arakoczy@gmail.com
 executables:
+- configure_aws.sh
+- create_stack.sh
+- delete_stack.sh
 - postgres_to_redshift
+- stack_env.sh
+- update_stack.sh
 extensions: []
 extra_rdoc_files: []
 files:
@@ -83,18 +88,26 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/configure_aws.sh
+- bin/create_stack.sh
+- bin/delete_stack.sh
 - bin/postgres_to_redshift
+- bin/stack_env.sh
+- bin/update_stack.sh
+- config/cloud-formation-local-postgres.json
+- config/cloud-formation-rds.json
 - lib/postgres_to_redshift.rb
 - lib/postgres_to_redshift/column.rb
 - lib/postgres_to_redshift/table.rb
 - lib/postgres_to_redshift/version.rb
 - postgres_to_redshift.gemspec
+- spec/features/small_table_spec.rb
 - spec/lib/postgres_to_redshift/column_spec.rb
 - spec/lib/postgres_to_redshift/table_spec.rb
 - spec/lib/postgres_to_redshift_spec.rb
 - spec/spec_helper.rb
 - spec/spec_prepare.rb
-homepage: https://github.com/kitchensurfing/postgres_to_redshift
+homepage: https://github.com/toothrot/postgres_to_redshift
 licenses:
 - MIT
 metadata: {}
@@ -119,8 +132,10 @@ signing_key:
 specification_version: 4
 summary: Load postgres databases into Amazon Redshift
 test_files:
+- spec/features/small_table_spec.rb
 - spec/lib/postgres_to_redshift/column_spec.rb
 - spec/lib/postgres_to_redshift/table_spec.rb
 - spec/lib/postgres_to_redshift_spec.rb
 - spec/spec_helper.rb
 - spec/spec_prepare.rb
+has_rdoc: