table_copy 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a22be48d16345c92700399d3cffdddff1df4378e
-  data.tar.gz: b7df0eafee1cd537a06931fd036a7663a0ebf539
+  metadata.gz: e1b6fbde1c068309a2bb36fb4aeba74a6f56efcc
+  data.tar.gz: bfafe52a0a0bf3792293cb9c8a58bae71a288b2a
 SHA512:
-  metadata.gz: f914d7745eb8e29085dcac211c7fef39dc03374bde943528d1e9037eb2dea78789be06b0e5b1c84600315189a02f2e3e18da7990584b7da7568f1216412e63ee
-  data.tar.gz: b927514b338af025f253de8cf6897ed1f3208d18e7b644bc678586332dea0e9a3022784773f4df216b44ef9d119920bfb52091e4b66937283be947fe1e68c9a1
+  metadata.gz: f875ce996caa3a33633af3a04368d2ce8d3a0ac0787cee6c1c13ba22ccf908869bde6dfb39f4b2def4ca20ba60ec15865e5ecd7e3fbcaabc83cbbb1cfdac6b9e
+  data.tar.gz: 5e193bcf30ec23fa9a53f11785e1fdcb990ad78c31fe51bd0faac515bfa378a2f394f0d18107248c4a73938de1e71954e311595f6c041219ebccefb775192071

data/lib/table_copy/copier.rb

@@ -23,17 +23,20 @@ module TableCopy
 
     def droppy
       logger.info { "Droppy #{destination_table.table_name}" }
+      views = destination_table.query_views
+
       destination_table.transaction do
-        views = destination_table.query_views
         destination_table.drop(cascade: true)
         create_table
-        destination_table.create_views(views)
-        logger.info { "Created #{views.count} views for #{destination_table.table_name}" }
         moved_count = destination_table.copy_data_from(source_table)
         logger.info { "#{moved_count} rows moved to #{destination_table.table_name}" }
         destination_table.create_indexes
         logger.info { "Completed #{source_table.indexes.count} indexes on #{destination_table.table_name}." }
       end
+
+      destination_table.create_views(views).each do |view_name, view_status|
+        logger.info { "#{view_status ? 'Created' : 'Failed to create'} view #{view_name} for #{destination_table.table_name}" }
+      end
     end
 
     def find_deletes

data/lib/table_copy/pg/destination.rb

@@ -109,8 +109,14 @@ module TableCopy
 
       def create_views(views)
         with_conn do |conn|
-          views.each do |view|
-            conn.exec("create or replace view #{view['viewname']} as (#{view['definition'].gsub(/;\z/, '')})")
+          views.inject({}) do |result, view|
+            begin
+              conn.exec("create or replace view #{view['viewname']} as (#{view['definition'].gsub(/;\z/, '')})")
+              result[view['viewname']] = true
+            rescue ::PG::UndefinedTable, ::PG::UndefinedColumn => e
+              result[view['viewname']] = false
+            end
+            result
           end
         end
       end

data/lib/table_copy/version.rb

@@ -1,3 +1,3 @@
 module TableCopy
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/spec/db.rb

@@ -0,0 +1,110 @@
+class DB
+  attr_reader :table_name, :view_name, :conn
+
+  def initialize(table_name: 'table_name', view_name: 'view_name', conn: $pg_conn)
+    @table_name = table_name
+    @view_name  = view_name
+    @conn       = conn
+  end
+
+  def with_conn
+    yield conn
+  end
+
+  def table_exists?(name=table_name)
+    conn.exec("select count(*) from pg_tables where tablename='#{name}'").first['count'] == '1'
+  end
+
+  def view_exists?(name=view_name)
+    conn.exec("select count(*) from pg_views where viewname='#{name}'").first['count'] == '1'
+  end
+
+  def insert_data(name=table_name)
+    conn.exec("insert into #{name} values(#{next_val}, 'foo', '{bar, baz}')")
+  end
+
+  def row_count(name=table_name)
+    conn.exec("select count(*) from #{name}").first['count'].to_i
+  end
+
+  def create_table(name=table_name)
+    conn.exec("create table #{name} (column1 integer, column2 varchar(123), column3 varchar(256)[])")
+  end
+
+  def create_view(name=view_name, t_name: table_name)
+    conn.exec("create view #{name} as (select * from #{t_name})")
+  end
+
+  def drop_table(name=table_name)
+    conn.exec("drop table if exists #{name} cascade")
+  end
+
+  def indexes(name=table_name)
+    conn.exec(indexes_sql(name))
+  end
+
+  def add_field(name, t_name: table_name)
+    conn.exec("alter table #{t_name} add #{name} #{data_types.sample}")
+  end
+
+  def drop_field(name, t_name: table_name)
+    conn.exec("alter table #{t_name} drop column #{name}")
+  end
+
+  def has_field?(name, t_name: table_name)
+    conn.exec("select #{name} from #{t_name}")
+    true
+  rescue PG::UndefinedColumn
+    false
+  end
+
+  def delete_row(name=table_name, pk=nil)
+    conn.exec("delete from #{name} where column1=#{pk || min_pk(name)}")
+  end
+
+  def exec(sql)
+    conn.exec(sql)
+  end
+
+  private
+
+  def min_pk(name)
+    conn.exec("select min(column1) from #{name}").first['min']
+  end
+
+  def next_val
+    @next_val ||= 0
+    @next_val += 1
+  end
+
+  def data_types
+    [
+      'integer',
+      'varchar(123)',
+      'varchar(256)[]'
+    ]
+  end
+
+  def indexes_sql(name)
+    <<-SQL
+      select
+          i.relname as index_name,
+          a.attname as column_name
+      from
+          pg_class t,
+          pg_class i,
+          pg_index ix,
+          pg_attribute a
+      where
+          t.oid = ix.indrelid
+          and i.oid = ix.indexrelid
+          and a.attrelid = t.oid
+          and a.attnum = ANY(ix.indkey)
+          and t.relkind = 'r'
+          and t.relname = '#{name}'
+      order by
+          t.relname,
+          i.relname;
+    SQL
+  end
+end

data/spec/lib/table_copy/copier_spec.rb

@@ -92,7 +92,7 @@ describe TableCopy::Copier do
         expect(destination).to receive(:drop).with(cascade: true)
         expect(source).to receive(:fields_ddl).and_return(fields_ddl)
         expect(destination).to receive(:create).with(fields_ddl)
-        expect(destination).to receive(:create_views).with('views')
+        expect(destination).to receive(:create_views).with('views').and_return([])
         expect(destination).to receive(:copy_data_from).with(source)
         expect(destination).to receive(:create_indexes)
         copier.droppy
@@ -122,7 +122,201 @@ describe TableCopy::Copier do
         end
       end
     end
-
   end
 
+  describe 'integration tests', speed: 'slow' do
+    let(:db1) { DB.new(conn: $pg_conn,  table_name: table_name1) }
+    let(:db2) { DB.new(conn: $pg_conn2, table_name: table_name2) }
+
+    let(:table_name1) { 'table_name1' }
+    let(:table_name2) { 'table_name2' }
+
+    let(:source) { TableCopy::PG::Source.new(
+      table_name: table_name1,
+      conn_method: db1.method(:with_conn)
+    ) }
+
+    let(:sequence_field) { 'column1' }
+
+    let(:dest) { TableCopy::PG::Destination.new(
+      table_name: table_name2,
+      conn_method: db2.method(:with_conn),
+      primary_key: 'column1',
+      sequence_field: sequence_field,
+      indexes: source.indexes,
+      fields:  source.fields
+    )}
+
+    let(:copier) { TableCopy::Copier.new(source, dest) }
+
+    before do
+      db1.create_table
+    end
+
+    after do
+      db1.drop_table
+    end
+
+    describe '#update' do
+      context 'no destination table' do
+        after { db2.drop_table }
+
+        it 'creates the table' do
+          expect {
+            copier.update
+          }.to change {
+            db2.table_exists?
+          }.from(false).to(true)
+        end
+      end
+
+      context 'destination table exists' do
+        before do
+          db2.create_table
+        end
+
+        after do
+          db2.drop_table
+        end
+
+        before do
+          db1.insert_data
+          db1.insert_data
+          db2.insert_data
+        end
+
+        context 'a max sequence is available' do
+          it 'updates the table with new data' do
+            expect {
+              copier.update
+            }.to change {
+              db2.row_count
+            }.from(1).to(2)
+          end
+        end
+
+        context 'no max sequence is available' do
+          let(:sequence_field) { nil }
+
+          it 'updates the table with new data' do
+            expect(destination.max_sequence).to eq nil
+
+            expect {
+              copier.update
+            }.to change {
+              db2.row_count
+            }.from(1).to(2)
+          end
+        end
+
+        context 'a field is added' do
+          let(:new_field) { 'new_field' }
+
+          before do
+            db1.add_field(new_field)
+          end
+
+          it 'adds the field to the destination table' do
+            expect {
+              copier.update
+            }.to change {
+              db2.has_field?(new_field)
+            }.from(false).to(true)
+          end
+        end
+      end
+    end
+
+    context 'within a transaction in the destination' do
+      before do
+        db2.create_table
+      end
+
+      after do
+        db2.drop_table
+      end
+
+      describe '#droppy' do
+        let(:new_field) { 'new_field' }
+
+        before do
+          db1.add_field(new_field)
+        end
+
+        it 'drops and rebuilds the destination table' do
+          expect {
+            copier.droppy
+          }.to change {
+            db2.has_field?(new_field)
+          }.from(false).to(true)
+        end
+
+        context 'with pre-existing views' do
+          before do
+            db2.create_view
+          end
+
+          it 'rebuilds views' do
+            expect {
+              copier.droppy
+            }.not_to change {
+              db2.view_exists?
+            }.from(true)
+          end
+
+          context 'a view becomes invalid' do
+            before do
+              db1.drop_field('column2')
+              db2.exec("create view view_name2 as (select column1 from #{db2.table_name})")
+            end
+
+            it 'rebuilds valid views' do
+              expect {
+                copier.droppy
+              }.to change {
+                db2.view_exists?
+              }.from(true).to(false)
+            end
+          end
+        end
+      end
+
+      context 'destination has rows absent from source' do
+        before { 3.times { db2.insert_data } }
+
+        describe '#find_deletes' do
+          it 'finds and removes deleted rows' do
+            expect {
+              copier.find_deletes
+            }.to change {
+              db2.row_count
+            }.from(3).to(0)
+          end
+        end
+
+        describe '#diffy' do
+          before do
+            5.times { db1.insert_data }
+            db1.delete_row
+          end
+
+          it 'copies data from source' do
+            expect {
+              copier.diffy
+            }.to change {
+              db2.row_count
+            }.from(3).to(4) # +2 -1
+          end
+
+          it 'finds and removes deleted rows' do
+            expect {
+              copier.diffy
+            }.to change {
+              db2.row_count
+            }.from(3).to(4) # +2 -1
+          end
+        end
+      end
+    end
+  end
 end

data/spec/lib/table_copy/pg/destination_spec.rb

@@ -2,59 +2,13 @@ require 'table_copy/pg/destination'
 require 'table_copy/pg/index'
 
 describe TableCopy::PG::Destination do
-  let(:conn) { $pg_conn }
   let(:table_name) { 'table_name' }
   let(:view_name)  { 'view_name' }
-  let(:indexes_sql) {
-    <<-SQL
-      select
-          i.relname as index_name,
-          a.attname as column_name
-      from
-          pg_class t,
-          pg_class i,
-          pg_index ix,
-          pg_attribute a
-      where
-          t.oid = ix.indrelid
-          and i.oid = ix.indexrelid
-          and a.attrelid = t.oid
-          and a.attnum = ANY(ix.indkey)
-          and t.relkind = 'r'
-          and t.relname = '#{table_name}'
-      order by
-          t.relname,
-          i.relname;
-    SQL
-  }
-
-  def with_conn
-    yield conn
-  end
-
-  def table_exists?(name=table_name)
-    conn.exec("select count(*) from pg_tables where tablename='#{name}'").first['count'] == '1'
-  end
-
-  def view_exists?(name=view_name)
-    conn.exec("select count(*) from pg_views where viewname='#{name}'").first['count'] == '1'
-  end
-
-  def insert_data(name=table_name)
-    conn.exec("insert into #{name} values(1, 'foo', '{bar, baz}')")
-  end
-
-  def row_count(name=table_name)
-    conn.exec("select count(*) from #{name}").first['count'].to_i
-  end
-
-  def create_table(name=table_name)
-    conn.exec("create table #{name} (column1 integer, column2 varchar(123), column3 varchar(256)[])")
-  end
+  let(:db) { DB.new(table_name: table_name, view_name: view_name) }
 
   let(:dest) { TableCopy::PG::Destination.new(
     table_name: table_name,
-    conn_method: method(:with_conn),
+    conn_method: db.method(:with_conn),
     indexes: [ TableCopy::PG::Index.new(table_name, nil, ['column1']) ],
     fields: [ 'column1', 'column2', 'column3' ],
     primary_key: 'column1',
@@ -62,7 +16,7 @@ describe TableCopy::PG::Destination do
   )}
 
   after do
-    conn.exec("drop table if exists #{table_name} cascade")
+    db.drop_table
   end
 
   describe '#to_s' do
@@ -73,7 +27,7 @@ describe TableCopy::PG::Destination do
 
   context 'a table exists' do
     before do
-      create_table
+      db.create_table
     end
 
     let(:expected_view) {
@@ -87,7 +41,7 @@ describe TableCopy::PG::Destination do
 
     context 'a view exists' do
       before do
-        conn.exec("create view #{view_name} as (select * from #{table_name})")
+        db.create_view
       end
 
       describe '#query_views' do
@@ -102,15 +56,35 @@ describe TableCopy::PG::Destination do
         expect {
           dest.create_views(expected_view)
         }.to change {
-          view_exists?
+          db.view_exists?
         }.from(false).to(true)
       end
+
+      it 'returns a hash of name => success' do
+        expect(dest.create_views(expected_view)).to eq({ view_name => true })
+      end
+
+      context 'a view fails to be created' do
+        let(:failing_view) { {
+          "viewname"   => 'another_view_name',
+          "definition" => "SELECT #{table_name}.column_does_no_exist FROM #{table_name};"
+        } }
+
+        let(:expected_result) { {
+          view_name => true,
+          'another_view_name' => false
+        } }
+
+        it 'returns a hash of name => success' do
+          expect(dest.create_views(expected_view << failing_view)).to eq(expected_result)
+        end
+      end
     end
 
     describe '#none?' do
       it 'indicates whether the table has any data' do
         expect {
-          insert_data
+          db.insert_data
         }.to change {
           dest.none?
         }.from(true).to(false)
@@ -122,10 +96,10 @@ describe TableCopy::PG::Destination do
         it 'opens and commits a transaction' do
           expect {
             dest.transaction do
-              insert_data
+              db.insert_data
             end
           }.to change {
-            conn.exec("select count(*) from #{table_name}").first['count'].to_i
+            db.row_count
           }.by(1)
         end
       end
@@ -135,12 +109,12 @@ describe TableCopy::PG::Destination do
           expect {
             begin
               dest.transaction do
-                insert_data
+                db.insert_data
                 raise
               end
             rescue RuntimeError; end
           }.not_to change {
-            conn.exec("select count(*) from #{table_name}").first['count'].to_i
+            db.row_count
           }
         end
       end
@@ -152,7 +126,7 @@ describe TableCopy::PG::Destination do
         expect {
           dest.drop
         }.to change {
-          table_exists?
+          db.table_exists?
         }.from(true).to(false)
       end
     end
@@ -162,7 +136,7 @@ describe TableCopy::PG::Destination do
         expect {
           dest.create_indexes
         }.to change {
-          conn.exec(indexes_sql).count
+          db.indexes.count
         }.from(0).to(1)
       end
     end
@@ -177,7 +151,7 @@ describe TableCopy::PG::Destination do
       context 'sequence field specified' do
         let(:dest) { TableCopy::PG::Destination.new(
           table_name: table_name,
-          conn_method: method(:with_conn),
+          conn_method: db.method(:with_conn),
           sequence_field: 'column1'
         )}
 
@@ -189,7 +163,7 @@ describe TableCopy::PG::Destination do
 
         context 'with rows' do
           before do
-            insert_data
+            db.insert_data
           end
 
           it 'returns the max value of the sequence field' do
@@ -215,25 +189,25 @@ describe TableCopy::PG::Destination do
             expect {
               dest.copy_data_from(source)
             }.to change {
-              row_count
+              db.row_count
             }.from(0).to(1)
           end
         end
 
         context 'temp is true' do
           before do
-            create_table("temp_#{table_name}")
+            db.create_table("temp_#{table_name}")
           end
 
           after do
-            conn.exec("drop table if exists temp_#{table_name}")
+            db.drop_table("temp_#{table_name}")
           end
 
           it 'inserts data into temp table' do
             expect {
               dest.copy_data_from(source, temp: true)
             }.to change {
-              row_count("temp_#{table_name}")
+              db.row_count("temp_#{table_name}")
             }.from(0).to(1)
           end
         end
@@ -250,7 +224,7 @@ describe TableCopy::PG::Destination do
           expect {
             dest.copy_data_from(source, pk_only: true)
           }.to change {
-            row_count
+            db.row_count
           }.from(0).to(1)
         end
       end
@@ -266,7 +240,7 @@ describe TableCopy::PG::Destination do
           expect {
             dest.copy_data_from(source, update: 'a_value')
           }.to change {
-            row_count
+            db.row_count
           }.from(0).to(1)
         end
       end
@@ -274,43 +248,43 @@ describe TableCopy::PG::Destination do
 
     context 'with temp table' do
       before do
-        create_table("temp_#{table_name}")
+        db.create_table("temp_#{table_name}")
       end
 
       after do
-        conn.exec("drop table if exists temp_#{table_name}")
+        db.drop_table("temp_#{table_name}")
       end
 
       describe '#copy_from_temp' do
         before do
-          insert_data("temp_#{table_name}")
+          db.insert_data("temp_#{table_name}")
         end
 
         it 'upserts from the temp table' do
           expect {
             dest.copy_from_temp
           }.to change {
-            row_count
+            db.row_count
           }.from(0).to(1)
 
           expect {
             dest.copy_from_temp
           }.not_to change {
-            row_count
+            db.row_count
           }.from(1)
         end
       end
 
       describe '#delete_not_in_temp' do
         before do
-          insert_data
+          db.insert_data
         end
 
         it 'deletes row that are not in the temp table' do
           expect {
             dest.delete_not_in_temp
           }.to change {
-            row_count
+            db.row_count
           }.from(1).to(0)
         end
       end
@@ -322,7 +296,7 @@ describe TableCopy::PG::Destination do
       expect {
         dest.create('column1 integer')
       }.to change {
-        table_exists?
+        db.table_exists?
       }.from(false).to(true)
     end
   end
@@ -333,10 +307,10 @@ describe TableCopy::PG::Destination do
         expect {
           dest.create_temp('column1 integer')
         }.to change {
-          table_exists?("temp_#{table_name}")
+          db.table_exists?("temp_#{table_name}")
         }.from(false).to(true)
       end
-      expect(table_exists?("temp_#{table_name}")).to eq false
+      expect(db.table_exists?("temp_#{table_name}")).to eq false
     end
   end
 end

data/spec/spec_helper.rb

@@ -3,6 +3,7 @@ SimpleCov.start
 
 require 'table_copy'
 require 'pg'
+require_relative 'db'
 # This file was generated by the `rspec --init` command. Conventionally, all
 # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
 # The generated `.rspec` file contains `--require spec_helper` which will cause this
@@ -24,12 +25,17 @@ RSpec.configure do |config|
     TableCopy.logger.level = Logger::FATAL
     system('dropdb table_copy_test')
     system('createdb table_copy_test')
-    $pg_conn = PG::Connection.new(YAML.load_file('config/database.yml'))
+    system('dropdb table_copy_test2')
+    system('createdb table_copy_test2')
+    $pg_conn = PG::Connection.new(dbname: 'table_copy_test')
+    $pg_conn2 = PG::Connection.new(dbname: 'table_copy_test2')
   end
 
   config.after(:suite) do
     $pg_conn.close
+    $pg_conn2.close
     system('dropdb table_copy_test')
+    system('dropdb table_copy_test2')
   end
 # The settings below are suggested to provide a good initial experience
 # with RSpec, but feel free to customize to your heart's content.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: table_copy
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - TLH
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-05 00:00:00.000000000 Z
+date: 2014-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -95,7 +95,6 @@ files:
 - README.md
 - Rakefile
 - bin/table_copy
-- config/database.yml
 - config/initializers/table_copy.rb.example
 - lib/table_copy.rb
 - lib/table_copy/copier.rb
@@ -105,6 +104,7 @@ files:
 - lib/table_copy/pg/index.rb
 - lib/table_copy/pg/source.rb
 - lib/table_copy/version.rb
+- spec/db.rb
 - spec/lib/table_copy/copier_spec.rb
 - spec/lib/table_copy/pg/destination_spec.rb
 - spec/lib/table_copy/pg/field_spec.rb
@@ -138,6 +138,7 @@ signing_key:
 specification_version: 4
 summary: Move full tables between databases.
 test_files:
+- spec/db.rb
 - spec/lib/table_copy/copier_spec.rb
 - spec/lib/table_copy/pg/destination_spec.rb
 - spec/lib/table_copy/pg/field_spec.rb

data/config/database.yml

@@ -1 +0,0 @@
-:dbname: table_copy_test