postgres_to_redshift 0.0.1 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 32377367b67f4405e131d319f810bbc38fbc2ac5
-  data.tar.gz: 97facb3cae58afb4c65b6cb8c17415e1e6e79f23
+  metadata.gz: b9aca81574e39acdc400fcfd77f454cb9c93f682
+  data.tar.gz: f4a8430406d25028ff1ac9b305837d34d0463464
 SHA512:
-  metadata.gz: bb62207056a659dce0b1cc6ceac28ee92af14794510b7c59150e5fe61d19e7f4e4c79fa8a1601989dfba3b097b0e0702bac67b258058a93f57d5f1c1dffd959f
-  data.tar.gz: e8f4cffccd43dbd33748c49b24bd4a0dadc670077ac798b2d5533da7cb144faa8ee60829fdbd60fcd34549c2af70b63d40edb05b71f730af9108680420140bde
+  metadata.gz: 8136aaffdd83cb676290393235f3cb1e99cf257c8c3dc193c0b234feec09b3f3cfedb29a8e6ad7f9f55641a2e1ef02c624f6d86073add5fec9ee5e0a2eacdde4
+  data.tar.gz: c0b9b49f1ba41e26fd2bb87328c5f672648a98ddd432335d8a899f6dbd722b1c32b6d6cb85b4de19630e42690eaf835957a6e801001d8a46972f917f0678482e

data/.gitignore

@@ -1,4 +1,5 @@
 /.bundle/
+/vendor/bundle
 /.yardoc
 /Gemfile.lock
 /_yardoc/

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+bundler_args: --without development --deployment --jobs=3 --retry=3
+cache: bundler
+rvm:
+  - 2.2.0
+  - 2.1.0
+before_script:
+  - psql -c 'create database travis_ci_test;' -U postgres
+env:
+  - POSTGRES_TO_REDSHIFT_SOURCE_URI=postgres://postgres@localhost/travis_ci_test

data/Gemfile

@@ -2,3 +2,8 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in postgres_to_redshift.gemspec
 gemspec
+
+group :test do
+  gem 'rake'
+  gem 'rspec'
+end

data/Gemfile.lock

@@ -0,0 +1,44 @@
+PATH
+  remote: .
+  specs:
+    postgres_to_redshift (0.1.1)
+      aws-sdk (~> 1.54)
+      pg (~> 0.17.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-sdk (1.61.0)
+      aws-sdk-v1 (= 1.61.0)
+    aws-sdk-v1 (1.61.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+    diff-lcs (1.2.5)
+    json (1.8.2)
+    mini_portile (0.6.2)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    pg (0.17.1)
+    rake (10.4.2)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.0)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.6)
+  postgres_to_redshift!
+  rake
+  rspec

data/README.md

@@ -2,6 +2,8 @@
 
 This gem copies data from postgres to redshift. It's especially useful to copy data from postgres to redshift in heroku.
 
+[![Build Status](https://travis-ci.org/kitchensurfing/postgres_to_redshift.svg?branch=master)](https://travis-ci.org/kitchensurfing/postgres_to_redshift)
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -20,8 +22,11 @@ Or install it yourself as:
 
 ## Usage
 
+Set your source and target databases, as well as your s3 intermediary.
+
 ```bash
-export REDSHIFT_URI='postgres://username:password@host:port/database-name'
+export POSTGRES_TO_REDSHIFT_SOURCE_URI='postgres://username:password@host:port/database-name'
+export POSTGRES_TO_REDSHIFT_TARGET_URI='postgres://username:password@host:port/database-name'
 export S3_DATABASE_EXPORT_ID='yourid'
 export S3_DATABASE_EXPORT_KEY='yourkey'
 export S3_DATABASE_EXPORT_BUCKET='some-bucket-to-use'

data/Rakefile

@@ -1,2 +1,10 @@
 require "bundler/gem_tasks"
 
+begin
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec)
+rescue LoadError
+end
+
+task(:default).clear
+task :default => :spec

data/bin/postgres_to_redshift

@@ -3,4 +3,3 @@
 require 'postgres_to_redshift'
 
 PostgresToRedshift.update_tables
-

data/lib/postgres_to_redshift/column.rb

@@ -0,0 +1,85 @@
+# table_catalog            | postgres_to_redshift
+# table_schema             | public
+# table_name               | films
+# column_name              | description
+# ordinal_position         | 2
+# column_default           |
+# is_nullable              | YES
+# data_type                | character varying
+# character_maximum_length | 255
+# character_octet_length   | 1020
+# numeric_precision        |
+# numeric_precision_radix  |
+# numeric_scale            |
+# datetime_precision       |
+# interval_type            |
+# interval_precision       |
+# character_set_catalog    |
+# character_set_schema     |
+# character_set_name       |
+# collation_catalog        |
+# collation_schema         |
+# collation_name           |
+# domain_catalog           |
+# domain_schema            |
+# domain_name              |
+# udt_catalog              | postgres_to_redshift
+# udt_schema               | pg_catalog
+# udt_name                 | varchar
+# scope_catalog            |
+# scope_schema             |
+# scope_name               |
+# maximum_cardinality      |
+# dtd_identifier           | 2
+# is_self_referencing      | NO
+# is_identity              | NO
+# identity_generation      |
+# identity_start           |
+# identity_increment       |
+# identity_maximum         |
+# identity_minimum         |
+# identity_cycle           |
+# is_generated             | NEVER
+# generation_expression    |
+# is_updatable             | YES
+#
+class PostgresToRedshift::Column
+  attr_accessor :attributes
+
+  CAST_TYPES_FOR_COPY = {
+    "text" => "CHARACTER VARYING(65535)",
+    "json" => "CHARACTER VARYING(65535)",
+    "bytea" => "CHARACTER VARYING(65535)",
+    "money" => "DECIMAL(19,2)",
+    "oid" => "CHARACTER VARYING(65535)",
+  }
+
+  def initialize(attributes: )
+    self.attributes = attributes
+  end
+
+  def name
+    attributes["column_name"]
+  end
+
+  def name_for_copy
+    if needs_type_cast?
+      %Q[CAST("#{name}" AS #{data_type_for_copy}) AS #{name}]
+    else
+      %Q["#{name}"]
+    end
+  end
+
+  def data_type
+    attributes["data_type"]
+  end
+
+  def data_type_for_copy
+    CAST_TYPES_FOR_COPY[data_type] || data_type
+  end
+
+  private
+  def needs_type_cast?
+    data_type != data_type_for_copy
+  end
+end

data/lib/postgres_to_redshift/table.rb

@@ -0,0 +1,54 @@
+# table_catalog                | postgres_to_redshift
+# table_schema                 | public
+# table_name                   | acquisition_pages
+# table_type                   | BASE TABLE
+# self_referencing_column_name |
+# reference_generation         |
+# user_defined_type_catalog    |
+# user_defined_type_schema     |
+# user_defined_type_name       |
+# is_insertable_into           | YES
+# is_typed                     | NO
+# commit_action                |
+#
+class PostgresToRedshift
+  class Table
+    attr_accessor :attributes, :columns
+
+    def initialize(attributes: , columns: [])
+      self.attributes = attributes
+      self.columns = columns
+    end
+
+    def name
+      attributes["table_name"]
+    end
+    alias_method :to_s, :name
+
+    def target_table_name
+      name.gsub(/_view$/, '')
+    end
+
+    def columns=(column_definitions = [])
+      @columns = column_definitions.map do |column_definition|
+        Column.new(attributes: column_definition)
+      end
+    end
+
+    def columns_for_create
+      columns.map do |column|
+        %Q["#{column.name}" #{column.data_type_for_copy}]
+      end.join(", ")
+    end
+
+    def columns_for_copy
+      columns.map do |column|
+        column.name_for_copy
+      end.join(", ")
+    end
+
+    def is_view?
+      attributes["table_type"] == "VIEW"
+    end
+  end
+end

data/lib/postgres_to_redshift/version.rb

@@ -1,3 +1,3 @@
 class PostgresToRedshift
-  VERSION = "0.0.1"
+  VERSION = "0.1.1"
 end

data/lib/postgres_to_redshift.rb

@@ -2,50 +2,74 @@ require "postgres_to_redshift/version"
 require 'pg'
 require 'uri'
 require 'aws-sdk'
+require 'zlib'
+require 'stringio'
+require "postgres_to_redshift/table"
+require "postgres_to_redshift/column"
 
 class PostgresToRedshift
+  class << self
+    attr_accessor :source_uri, :target_uri
+  end
+
   attr_reader :source_connection, :target_connection, :s3
 
   def self.update_tables
-    update_tables = PostgresToRedshift.new(source_uri: ARGV[0])
-    update_tables.create_new_tables
+    update_tables = PostgresToRedshift.new
+
+    update_tables.tables.each do |table|
+      target_connection.exec("CREATE TABLE IF NOT EXISTS public.#{table.target_table_name} (#{table.columns_for_create})")
+
+      update_tables.copy_table(table)
+
+      update_tables.import_table(table)
+    end
+  end
+
+  def self.source_uri
+    @source_uri ||= URI.parse(ENV['POSTGRES_TO_REDSHIFT_SOURCE_URI'])
+  end
+
+  def self.target_uri
+    @target_uri ||= URI.parse(ENV['POSTGRES_TO_REDSHIFT_TARGET_URI'])
+  end
+
+  def self.source_connection
+    unless instance_variable_defined?(:"@source_connection")
+      @source_connection = PG::Connection.new(host: source_uri.host, port: source_uri.port, user: source_uri.user || ENV['USER'], password: source_uri.password, dbname: source_uri.path[1..-1])
+      @source_connection.exec("SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY;")
+    end
+
+    @source_connection
+  end
 
-    # FIXME: BIG WARNING HERE: this order is important. We want the views to overwrite the tables. We should make it so the order doesn't matter later.
-    update_tables.copy_tables
-    update_tables.copy_views
-    update_tables.import_tables
+  def self.target_connection
+    unless instance_variable_defined?(:"@target_connection")
+      @target_connection = PG::Connection.new(host: target_uri.host, port: target_uri.port, user: target_uri.user || ENV['USER'], password: target_uri.password, dbname: target_uri.path[1..-1])
+    end
+
+    @target_connection
   end
 
-  def initialize(source_uri:)
-    source_uri = URI.parse(source_uri)
-    target_uri = URI.parse(ENV['REDSHIFT_URI'])
-    @source_connection = PG::Connection.new(host: source_uri.host, port: source_uri.port, user: source_uri.user, password: source_uri.password, dbname: source_uri.path[1..-1])
-    @source_connection.exec("SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY;")
-    @target_connection = PG::Connection.new(host: target_uri.host, port: target_uri.port, user: target_uri.user, password: target_uri.password, dbname: target_uri.path[1..-1])
+  def source_connection
+    self.class.source_connection
   end
 
-  def views
-    source_connection.exec("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'VIEW'").map { |row| row["table_name"] } - ["pg_stat_statements"]
+  def target_connection
+    self.class.target_connection
   end
 
   def tables
-    source_connection.exec("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'").map { |row| row["table_name"] }
+    source_connection.exec("SELECT * FROM information_schema.tables WHERE table_schema = 'public' AND table_type in ('BASE TABLE', 'VIEW')").map do |table_attributes|
+      table = Table.new(attributes: table_attributes)
+      next if table.name =~ /^pg_/
+      table.columns = column_definitions(table)
+      table
+    end.compact
   end
 
-  def table_columns(table_name)
-    source_connection.exec("SELECT column_name, data_type, character_maximum_length FROM information_schema.columns WHERE table_schema='public' AND table_name='#{table_name}'").map do |row| 
-      data_type = row["data_type"]
-      data_type.gsub!(/text/, 'character varying(max)')
-      data_type.gsub!(/json/, 'character varying(max)')
-      data_type.gsub!(/bytea/, 'character varying(max)')
-      data_type.gsub!(/money/, 'character varying(max)')
-
-      if row["character_maximum_length"].to_s.length > 0
-        %Q|"#{row["column_name"]}" #{data_type}(#{row["character_maximum_length"]})|
-      else
-        %Q|"#{row["column_name"]}" #{data_type}|
-      end
-    end.join(", ")
+  def column_definitions(table)
+    source_connection.exec("SELECT * FROM information_schema.columns WHERE table_schema='public' AND table_name='#{table.name}' order by ordinal_position")
   end
 
   def s3
@@ -56,68 +80,41 @@ class PostgresToRedshift
     @bucket ||= s3.buckets[ENV['S3_DATABASE_EXPORT_BUCKET']]
   end
 
-  def create_new_tables
-    tables.each do |table|
-      target_connection.exec("CREATE TABLE IF NOT EXISTS public.#{table} (#{table_columns(table)})")
-    end
-  end
+  def copy_table(table)
+    buffer = StringIO.new
+    zip = Zlib::GzipWriter.new(buffer)
 
-  def copy_table(source_table, target_table, is_view = false)
-    buffer = ""
-    puts "Downloading #{source_table}"
-    copy_command = 
-      if is_view
-        "COPY (SELECT * FROM #{source_table}) TO STDOUT WITH DELIMITER '|'"
-      else
-        "COPY #{source_table} TO STDOUT WITH DELIMITER '|'"
-      end
+    puts "Downloading #{table}"
+    copy_command = "COPY (SELECT #{table.columns_for_copy} FROM #{table.name}) TO STDOUT WITH DELIMITER '|'"
 
     source_connection.copy_data(copy_command) do
       while row = source_connection.get_copy_data
-        buffer << row
+        zip.write(row)
       end
     end
-    upload_table(target_table, buffer)
+    zip.finish
+    buffer.rewind
+    upload_table(table, buffer)
   end
 
-  def upload_table(target_table, buffer)
-    puts "Uploading #{target_table}"
-    bucket.objects["export/#{target_table}.psv"].delete
-    bucket.objects["export/#{target_table}.psv"].write(buffer, acl: :authenticated_read)
+  def upload_table(table, buffer)
+    puts "Uploading #{table.target_table_name}"
+    bucket.objects["export/#{table.target_table_name}.psv.gz"].delete
+    bucket.objects["export/#{table.target_table_name}.psv.gz"].write(buffer, acl: :authenticated_read)
   end
 
-  def import_table(target_table)
-    puts "Importing #{target_table}"
-    target_connection.exec("DROP TABLE IF EXISTS public.#{target_table}_updating")
+  def import_table(table)
+    puts "Importing #{table.target_table_name}"
+    target_connection.exec("DROP TABLE IF EXISTS public.#{table.target_table_name}_updating")
 
     target_connection.exec("BEGIN;")
 
-    target_connection.exec("ALTER TABLE public.#{target_table} RENAME TO #{target_table}_updating")
+    target_connection.exec("ALTER TABLE public.#{table.target_table_name} RENAME TO #{table.target_table_name}_updating")
 
-    target_connection.exec("CREATE TABLE public.#{target_table} (#{table_columns(target_table)})")
+    target_connection.exec("CREATE TABLE public.#{table.target_table_name} (#{table.columns_for_create})")
 
-    target_connection.exec("COPY public.#{target_table} FROM 's3://#{ENV['S3_DATABASE_EXPORT_BUCKET']}/export/#{target_table}.psv' CREDENTIALS 'aws_access_key_id=#{ENV['S3_DATABASE_EXPORT_ID']};aws_secret_access_key=#{ENV['S3_DATABASE_EXPORT_KEY']}' TRUNCATECOLUMNS ESCAPE DELIMITER as '|';")
+    target_connection.exec("COPY public.#{table.target_table_name} FROM 's3://#{ENV['S3_DATABASE_EXPORT_BUCKET']}/export/#{table.target_table_name}.psv.gz' CREDENTIALS 'aws_access_key_id=#{ENV['S3_DATABASE_EXPORT_ID']};aws_secret_access_key=#{ENV['S3_DATABASE_EXPORT_KEY']}' GZIP TRUNCATECOLUMNS ESCAPE DELIMITER as '|';")
 
     target_connection.exec("COMMIT;")
   end
-
-  def copy_tables
-    tables.each do |table|
-      copy_table(table, table)
-    end
-  end
-
-  def copy_views
-    views.each do |view|
-      table = view.gsub(/_view/, '')
-      copy_table(view, table, true)
-    end
-  end
-
-  # FIXME: This relies on views being uploaded after tables.
-  def import_tables
-    tables.each do |table|
-      import_table(table)
-    end
-  end
 end

data/postgres_to_redshift.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_dependency "pg", "~> 0.17.0"
   spec.add_dependency "aws-sdk", "~> 1.54"

data/spec/lib/postgres_to_redshift/column_spec.rb

@@ -0,0 +1,168 @@
+require 'spec_helper'
+
+RSpec.describe PostgresToRedshift::Column do
+  context 'with a simple column' do
+    before do
+      attributes = {
+        "table_catalog"            => "postgres_to_redshift",
+        "table_schema"             => "public",
+        "table_name"               => "films",
+        "column_name"              => "description",
+        "ordinal_position"         => "2",
+        "column_default"           => nil,
+        "is_nullable"              => "YES",
+        "data_type"                => "character varying",
+        "character_maximum_length" => "255",
+        "character_octet_length"   => "1020"
+      }
+
+      @column = PostgresToRedshift::Column.new attributes: attributes
+    end
+
+    describe '#name' do
+      it 'returns the column name' do
+        expect(@column.name).to eq("description")
+      end
+    end
+  end
+
+  describe '#name_for_copy' do
+    it 'casts fields to appropriate type' do
+      attributes = {
+        "table_catalog"            => "postgres_to_redshift",
+        "table_schema"             => "public",
+        "table_name"               => "films",
+        "column_name"              => "description",
+        "ordinal_position"         => "2",
+        "column_default"           => nil,
+        "is_nullable"              => "YES",
+        "data_type"                => "text",
+        "character_maximum_length" => nil,
+        "character_octet_length"   => "1073741824"
+      }
+
+      column = PostgresToRedshift::Column.new attributes: attributes
+      expect(column.name_for_copy).to eq('CAST("description" AS CHARACTER VARYING(65535)) AS description')
+    end
+
+    it 'does not cast fields that do not need casting' do
+      attributes = {
+        "table_catalog"            => "postgres_to_redshift",
+        "table_schema"             => "public",
+        "table_name"               => "films",
+        "column_name"              => "description",
+        "ordinal_position"         => "2",
+        "column_default"           => nil,
+        "is_nullable"              => "YES",
+        "data_type"                => "character varying",
+        "character_maximum_length" => "255",
+        "character_octet_length"   => "1020"
+      }
+
+      column = PostgresToRedshift::Column.new attributes: attributes
+      expect(column.name_for_copy).to eq('"description"')
+    end
+  end
+
+  describe "#data_type_for_copy" do
+    it 'casts text to character varying(65535)' do
+      attributes = {
+        "table_catalog"            => "postgres_to_redshift",
+        "table_schema"             => "public",
+        "table_name"               => "films",
+        "column_name"              => "description",
+        "ordinal_position"         => "2",
+        "column_default"           => nil,
+        "is_nullable"              => "YES",
+        "data_type"                => "text",
+        "character_maximum_length" => nil,
+        "character_octet_length"   => "1073741824"
+      }
+
+      column = PostgresToRedshift::Column.new attributes: attributes
+      expect(column.data_type_for_copy).to eq("CHARACTER VARYING(65535)")
+    end
+
+    it 'casts json to character varying(65535)' do
+      attributes = {
+        "table_catalog"            => "postgres_to_redshift",
+        "table_schema"             => "public",
+        "table_name"               => "films",
+        "column_name"              => "description",
+        "ordinal_position"         => "2",
+        "column_default"           => nil,
+        "is_nullable"              => "YES",
+        "data_type"                => "json",
+      }
+
+      column = PostgresToRedshift::Column.new attributes: attributes
+      expect(column.data_type_for_copy).to eq("CHARACTER VARYING(65535)")
+    end
+
+    it 'casts bytea to character varying(65535)' do
+      attributes = {
+        "table_catalog"            => "postgres_to_redshift",
+        "table_schema"             => "public",
+        "table_name"               => "films",
+        "column_name"              => "description",
+        "ordinal_position"         => "2",
+        "column_default"           => nil,
+        "is_nullable"              => "YES",
+        "data_type"                => "bytea",
+      }
+
+      column = PostgresToRedshift::Column.new attributes: attributes
+      expect(column.data_type_for_copy).to eq("CHARACTER VARYING(65535)")
+    end
+
+    it 'casts money to decimal(19,2)' do
+      attributes = {
+        "table_catalog"            => "postgres_to_redshift",
+        "table_schema"             => "public",
+        "table_name"               => "films",
+        "column_name"              => "description",
+        "ordinal_position"         => "2",
+        "column_default"           => nil,
+        "is_nullable"              => "YES",
+        "data_type"                => "money",
+      }
+
+      column = PostgresToRedshift::Column.new attributes: attributes
+      expect(column.data_type_for_copy).to eq("DECIMAL(19,2)")
+    end
+
+    it 'casts oid to character varying' do
+      attributes = {
+        "table_catalog"            => "postgres_to_redshift",
+        "table_schema"             => "public",
+        "table_name"               => "films",
+        "column_name"              => "description",
+        "ordinal_position"         => "2",
+        "column_default"           => nil,
+        "is_nullable"              => "YES",
+        "data_type"                => "oid",
+      }
+
+      column = PostgresToRedshift::Column.new attributes: attributes
+      expect(column.data_type_for_copy).to eq("CHARACTER VARYING(65535)")
+    end
+
+    it "returns the data type if no cast necessary" do
+      attributes = {
+        "table_catalog"            => "postgres_to_redshift",
+        "table_schema"             => "public",
+        "table_name"               => "films",
+        "column_name"              => "description",
+        "ordinal_position"         => "2",
+        "column_default"           => nil,
+        "is_nullable"              => "YES",
+        "data_type"                => "character varying",
+        "character_maximum_length" => "255",
+        "character_octet_length"   => "1020"
+      }
+
+      column = PostgresToRedshift::Column.new attributes: attributes
+      expect(column.data_type_for_copy).to eq("character varying")
+    end
+  end
+end

data/spec/lib/postgres_to_redshift/table_spec.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+
+RSpec.describe PostgresToRedshift::Table do
+  context 'with a simple table' do
+    before do
+      attributes = { 
+        "table_catalog" => "postgres_to_redshift",
+        "table_schema" => "public",
+        "table_name" => "films",
+        "table_type" => "BASE TABLE",
+      }
+      columns = [
+        {
+          "table_catalog"            => "postgres_to_redshift",
+          "table_schema"             => "public",
+          "table_name"               => "films",
+          "column_name"              => "description",
+          "ordinal_position"         => "2",
+          "column_default"           => nil,
+          "is_nullable"              => "YES",
+          "data_type"                => "character varying",
+          "character_maximum_length" => "255",
+          "character_octet_length"   => "1020"
+        }
+      ]
+
+      @table = PostgresToRedshift::Table.new(attributes: attributes, columns: columns)
+    end
+
+    describe '#name' do
+      it 'returns the name of the table' do
+        expect(@table.name).to eq("films")
+      end
+    end
+
+    describe '#columns' do
+      it 'returns a list of columns' do
+        expect(@table.columns.size).to eq(1)
+        expect(@table.columns.first.name).to eq("description")
+      end
+    end
+  end
+
+  describe '#is_view?' do
+    it 'returns true if it is a view' do
+      attributes = { 
+        "table_catalog" => "postgres_to_redshift",
+        "table_schema" => "public",
+        "table_name" => "films",
+        "table_type" => "VIEW",
+      }
+
+      table = PostgresToRedshift::Table.new(attributes: attributes)
+      expect(table.is_view?).to be_truthy
+    end
+
+    it 'returns false if it is not a view' do
+      attributes = { 
+        "table_catalog" => "postgres_to_redshift",
+        "table_schema" => "public",
+        "table_name" => "films",
+        "table_type" => "BASE TABLE",
+      }
+
+      table = PostgresToRedshift::Table.new(attributes: attributes)
+      expect(table.is_view?).to be_falsey
+    end
+  end
+
+  describe 'target_table_name' do
+    it 'strips _view from the end of the table name' do
+      attributes = { 
+        "table_catalog" => "postgres_to_redshift",
+        "table_schema" => "public",
+        "table_name" => "films_view",
+        "table_type" => "VIEW",
+      }
+
+      table = PostgresToRedshift::Table.new(attributes: attributes)
+      expect(table.target_table_name).to eq("films")
+    end
+  end
+end

data/spec/lib/postgres_to_redshift_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+RSpec.describe PostgresToRedshift do
+  it 'opens a read only connection to source database' do
+    read_only_state = PostgresToRedshift.source_connection.exec("SHOW transaction_read_only").first["transaction_read_only"]
+
+    expect(read_only_state).to eq("on")
+  end
+
+  context 'with a simple table' do
+    before do
+      PostgresToRedshift::Test.test_connection.exec(%Q[DROP TABLE IF EXISTS "films"; CREATE TABLE IF NOT EXISTS "films" ("id" SERIAL PRIMARY KEY, "title" text);])
+    end
+
+    it 'lists available tables' do
+      expect(PostgresToRedshift.new.tables.size).to eq(1)
+      expect(PostgresToRedshift.new.tables.first.name).to eq("films")
+    end
+
+    it 'lists column definitions' do
+      table = PostgresToRedshift.new.tables.first
+      film_columns = PostgresToRedshift.new.column_definitions(table)
+
+      expect(film_columns.to_a.size).to eq(2)
+      expect(film_columns.first["column_name"]).to eq("id")
+      expect(table.columns.first.name).to eq("id")
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,90 @@
+require File.expand_path("../../lib/postgres_to_redshift", __FILE__)
+require 'spec_prepare'
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause this
+# file to always be loaded, without a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    # be_bigger_than(2).and_smaller_than(4).description
+    #   # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #   # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Limits the available syntax to the non-monkey patched syntax that is recommended.
+  # For more details, see:
+  #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+  #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+end

data/spec/spec_prepare.rb

@@ -0,0 +1,15 @@
+module PostgresToRedshift::Test
+  def self.source_uri
+    PostgresToRedshift.source_uri
+  end
+
+  def self.test_connection
+    @test_connection ||= PG::Connection.new(host: source_uri.host, port: source_uri.port, user: source_uri.user || ENV['USER'], password: source_uri.password, dbname: source_uri.path[1..-1])
+  end
+end
+
+RSpec.configure do |config|
+  config.before :suite do
+    PostgresToRedshift::Test.test_connection
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postgres_to_redshift
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.1
 platform: ruby
 authors:
 - Alex Rakoczy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-20 00:00:00.000000000 Z
+date: 2015-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -76,14 +76,24 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
+- ".travis.yml"
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/postgres_to_redshift
 - lib/postgres_to_redshift.rb
+- lib/postgres_to_redshift/column.rb
+- lib/postgres_to_redshift/table.rb
 - lib/postgres_to_redshift/version.rb
 - postgres_to_redshift.gemspec
+- spec/lib/postgres_to_redshift/column_spec.rb
+- spec/lib/postgres_to_redshift/table_spec.rb
+- spec/lib/postgres_to_redshift_spec.rb
+- spec/spec_helper.rb
+- spec/spec_prepare.rb
 homepage: https://github.com/kitchensurfing/postgres_to_redshift
 licenses:
 - MIT
@@ -108,4 +118,9 @@ rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Load postgres databases into Amazon Redshift
-test_files: []
+test_files:
+- spec/lib/postgres_to_redshift/column_spec.rb
+- spec/lib/postgres_to_redshift/table_spec.rb
+- spec/lib/postgres_to_redshift_spec.rb
+- spec/spec_helper.rb
+- spec/spec_prepare.rb