table_copy 0.0.5

data/lib/table_copy.rb

@@ -0,0 +1,54 @@
+require 'logger'
+require 'table_copy/copier'
+
+module TableCopy
+  class << self
+    attr_writer :logger
+
+    def logger
+      @logger ||= Logger.new($stdout)
+    end
+
+    def links
+      if configured?
+        @links
+      else
+        configure_links
+        @links
+      end
+    end
+
+    def deferred_config(&block)
+      @deferred_config = block
+    end
+
+    def add_link(name, source, destination)
+      links_to_add[name] = TableCopy::Copier.new(source, destination)
+    end
+
+    private
+
+    def configure_links
+      synchronized do
+        return @links if configured?
+        @deferred_config.call if @deferred_config
+        @links = links_to_add
+      end
+    end
+
+    def configured?
+      @links && !@links.empty?
+    end
+
+    def links_to_add
+      @links_to_add ||= {}
+    end
+
+    def synchronized
+      @semaphore ||= Mutex.new
+      @semaphore.synchronize do
+        yield
+      end
+    end
+  end
+end

data/spec/lib/table_copy/copier_spec.rb

@@ -0,0 +1,126 @@
+require 'table_copy/copier'
+require 'table_copy/pg/source'
+require 'table_copy/pg/destination'
+
+describe TableCopy::Copier do
+  let(:copier) { TableCopy::Copier.new(source, destination) }
+  let(:source) { TableCopy::PG::Source.new({}) }
+  let(:destination) { TableCopy::PG::Destination.new({}) }
+  let(:fields_ddl) { 'fake fields ddl' }
+
+  describe '#update' do
+    context 'no destination table' do
+      it 'calls droppy' do
+        expect(destination).to receive(:none?).and_return(true)
+        expect(copier).to receive(:droppy)
+        copier.update
+      end
+    end
+
+    context 'destination table exists' do
+      before do
+        expect(destination).to receive(:none?).and_return(false)
+      end
+
+      context 'a max sequence is available' do
+        before do
+          expect(destination).to receive(:max_sequence).and_return(2345)
+        end
+
+        it 'updates the table with new data' do
+          expect(destination).to receive(:transaction).and_yield
+          expect(source).to receive(:fields_ddl).and_return(fields_ddl)
+          expect(destination).to receive(:create_temp).with(fields_ddl)
+          expect(destination).to receive(:copy_data_from).with(source, temp: true, update: 2345)
+          expect(destination).to receive(:copy_from_temp).with(except: nil)
+          copier.update
+        end
+      end
+
+      context 'no max sequence is available' do
+        before do
+          expect(destination).to receive(:max_sequence).and_return(nil)
+        end
+
+        it 'calls diffy_update' do
+          expect(destination).to receive(:transaction).and_yield
+          expect(source).to receive(:fields_ddl).and_return(fields_ddl)
+          expect(destination).to receive(:create_temp).with(fields_ddl)
+          expect(destination).to receive(:copy_data_from).with(source, temp: true)
+          expect(destination).to receive(:copy_from_temp)
+          copier.update
+        end
+      end
+    end
+
+    context 'PG::UndefinedTable is raised' do
+      before do
+        expect(destination).to receive(:none?).and_raise(PG::UndefinedTable, 'Intentionally raised.').once
+        expect(destination).to receive(:none?).and_return(true) # handle the retry
+      end
+
+      it "passes the source's ddl to #create on the destination" do
+        expect(source).to receive(:fields_ddl).and_return(fields_ddl)
+        expect(destination).to receive(:create).with(fields_ddl)
+        expect(copier).to receive(:droppy) # handle the retry
+        copier.update
+      end
+    end
+
+    context 'PG::UndefinedColumn is raised' do
+      let(:fields_ddl) { 'fake fields ddl' }
+
+      before do
+        expect(destination).to receive(:none?).and_raise(PG::UndefinedColumn, 'Intentionally raised.').once
+      end
+
+      it 'calls droppy' do
+        expect(copier).to receive(:droppy)
+        copier.update
+      end
+    end
+  end
+
+  context 'within a transaction in the destination' do
+    before do
+      expect(destination).to receive(:transaction).and_yield
+    end
+
+    describe '#droppy' do
+      it 'drops and rebuilds the destination table' do
+        expect(destination).to receive(:drop).with(cascade: true)
+        expect(source).to receive(:fields_ddl).and_return(fields_ddl)
+        expect(destination).to receive(:create).with(fields_ddl)
+        expect(destination).to receive(:copy_data_from).with(source)
+        expect(destination).to receive(:create_indexes)
+        copier.droppy
+      end
+    end
+
+    context 'after creating a temp table' do
+      before do
+        expect(source).to receive(:fields_ddl).and_return(fields_ddl)
+        expect(destination).to receive(:create_temp).with(fields_ddl)
+      end
+
+      describe '#find_deletes' do
+        it 'finds and removes deleted rows' do
+          expect(destination).to receive(:copy_data_from).with(source, temp: true, pk_only: true)
+          expect(destination).to receive(:delete_not_in_temp)
+          copier.find_deletes
+        end
+      end
+
+      describe '#diffy' do
+        it 'copies data form temp and finds and removes deleted rows' do
+          expect(destination).to receive(:copy_data_from).with(source, temp: true)
+          expect(destination).to receive(:copy_from_temp)
+          expect(destination).to receive(:delete_not_in_temp)
+          copier.diffy
+        end
+      end
+    end
+
+  end
+
+end

data/spec/lib/table_copy/pg/destination_spec.rb

@@ -0,0 +1,305 @@
+require 'table_copy/pg/destination'
+require 'table_copy/pg/index'
+
+describe TableCopy::PG::Destination do
+  let(:conn) { $pg_conn }
+  let(:table_name) { 'table_name' }
+  let(:indexes_sql) {
+    <<-SQL
+      select
+          i.relname as index_name,
+          a.attname as column_name
+      from
+          pg_class t,
+          pg_class i,
+          pg_index ix,
+          pg_attribute a
+      where
+          t.oid = ix.indrelid
+          and i.oid = ix.indexrelid
+          and a.attrelid = t.oid
+          and a.attnum = ANY(ix.indkey)
+          and t.relkind = 'r'
+          and t.relname = '#{table_name}'
+      order by
+          t.relname,
+          i.relname;
+    SQL
+  }
+
+  def with_conn
+    yield conn
+  end
+
+  def table_exists?(name=table_name)
+    conn.exec("select count(*) from pg_tables where tablename='#{name}'").first['count'] == '1'
+  end
+
+  def insert_data(name=table_name)
+    conn.exec("insert into #{name} values(1, 'foo', '{bar, baz}')")
+  end
+
+  def row_count(name=table_name)
+    conn.exec("select count(*) from #{name}").first['count'].to_i
+  end
+
+  def create_table(name=table_name)
+    conn.exec("create table #{name} (column1 integer, column2 varchar(123), column3 varchar(256)[])")
+  end
+
+  let(:dest) { TableCopy::PG::Destination.new(
+    table_name: table_name,
+    conn_method: method(:with_conn),
+    indexes: [ TableCopy::PG::Index.new(table_name, nil, ['column1']) ],
+    fields: [ 'column1', 'column2', 'column3' ],
+    primary_key: 'column1',
+    sequence_field: 'column1'
+  )}
+
+  after do
+    conn.exec("drop table if exists #{table_name}")
+  end
+
+  describe '#to_s' do
+    it 'returns the table name' do
+      expect(dest.to_s).to eq table_name
+    end
+  end
+
+  context 'a table exists' do
+    before do
+      create_table
+    end
+
+    describe '#none?' do
+      it 'indicates whether the table has any data' do
+        expect {
+          insert_data
+        }.to change {
+          dest.none?
+        }.from(true).to(false)
+      end
+    end
+
+    describe '#transaction' do
+      context 'no error is raised' do
+        it 'opens and commits a transaction' do
+          expect {
+            dest.transaction do
+              insert_data
+            end
+          }.to change {
+            conn.exec("select count(*) from #{table_name}").first['count'].to_i
+          }.by(1)
+        end
+      end
+
+      context 'an error is raised' do
+        it 'opens but does not commit a transaction' do
+          expect {
+            begin
+              dest.transaction do
+                insert_data
+                raise
+              end
+            rescue RuntimeError; end
+          }.not_to change {
+            conn.exec("select count(*) from #{table_name}").first['count'].to_i
+          }
+        end
+      end
+    end
+
+    describe '#drop' do
+      it 'drops a table' do
+        expect {
+          dest.drop
+        }.to change {
+          table_exists?
+        }.from(true).to(false)
+      end
+    end
+
+    describe '#create_indexes' do
+      it 'creates indexes' do
+        expect {
+          dest.create_indexes
+        }.to change {
+          conn.exec(indexes_sql).count
+        }.from(0).to(1)
+      end
+    end
+
+    describe '#max_sequence' do
+      context 'no sequence field' do
+        it 'returns nil' do
+          expect(dest.max_sequence).to be_nil
+        end
+      end
+
+      context 'sequence field specified' do
+        let(:dest) { TableCopy::PG::Destination.new(
+          table_name: table_name,
+          conn_method: method(:with_conn),
+          sequence_field: 'column1'
+        )}
+
+        context 'no rows' do
+          it 'returns nil' do
+            expect(dest.max_sequence).to be_nil
+          end
+        end
+
+        context 'with rows' do
+          before do
+            insert_data
+          end
+
+          it 'returns the max value of the sequence field' do
+            expect(dest.max_sequence).to eq '1'
+          end
+        end
+      end
+    end
+
+    describe '#copy_data_from' do
+      let(:source) { TableCopy::PG::Source.new({}) }
+      let(:source_conn) { double }
+
+      context 'all fields and rows' do
+        before do
+          expect(source).to receive(:copy_from).with('column1, column2, column3', nil).and_yield(source_conn)
+          expect(source_conn).to receive(:get_copy_data).and_return("1,foo,\"{bar,baz}\"\n")
+          expect(source_conn).to receive(:get_copy_data).and_return(nil)
+        end
+
+        context 'default options' do
+          it 'inserts data' do
+            expect {
+              dest.copy_data_from(source)
+            }.to change {
+              row_count
+            }.from(0).to(1)
+          end
+        end
+
+        context 'temp is true' do
+          before do
+            create_table("temp_#{table_name}")
+          end
+
+          after do
+            conn.exec("drop table if exists temp_#{table_name}")
+          end
+
+          it 'inserts data into temp table' do
+            expect {
+              dest.copy_data_from(source, temp: true)
+            }.to change {
+              row_count("temp_#{table_name}")
+            }.from(0).to(1)
+          end
+        end
+      end
+
+      context 'pk_only is true' do
+        before do
+          expect(source).to receive(:copy_from).with('column1', nil).and_yield(source_conn)
+          expect(source_conn).to receive(:get_copy_data).and_return("1\n")
+          expect(source_conn).to receive(:get_copy_data).and_return(nil)
+        end
+
+        it 'inserts data' do
+          expect {
+            dest.copy_data_from(source, pk_only: true)
+          }.to change {
+            row_count
+          }.from(0).to(1)
+        end
+      end
+
+      context 'update value is given' do
+        before do
+          expect(source).to receive(:copy_from).with('column1, column2, column3', "where column1 > 'a_value'").and_yield(source_conn)
+          expect(source_conn).to receive(:get_copy_data).and_return("1,foo,\"{bar,baz}\"\n")
+          expect(source_conn).to receive(:get_copy_data).and_return(nil)
+        end
+
+        it 'inserts data' do
+          expect {
+            dest.copy_data_from(source, update: 'a_value')
+          }.to change {
+            row_count
+          }.from(0).to(1)
+        end
+      end
+    end
+
+    context 'with temp table' do
+      before do
+        create_table("temp_#{table_name}")
+      end
+
+      after do
+        conn.exec("drop table if exists temp_#{table_name}")
+      end
+
+      describe '#copy_from_temp' do
+        before do
+          insert_data("temp_#{table_name}")
+        end
+
+        it 'upserts from the temp table' do
+          expect {
+            dest.copy_from_temp
+          }.to change {
+            row_count
+          }.from(0).to(1)
+
+          expect {
+            dest.copy_from_temp
+          }.not_to change {
+            row_count
+          }.from(1)
+        end
+      end
+
+      describe '#delete_not_in_temp' do
+        before do
+          insert_data
+        end
+
+        it 'deletes row that are not in the temp table' do
+          expect {
+            dest.delete_not_in_temp
+          }.to change {
+            row_count
+          }.from(1).to(0)
+        end
+      end
+    end
+  end
+
+  describe '#create' do
+    it 'creates a table' do
+      expect {
+        dest.create('column1 integer')
+      }.to change {
+        table_exists?
+      }.from(false).to(true)
+    end
+  end
+
+  describe '#create_temp' do
+    it 'creates a temporary table' do
+      dest.transaction do
+        expect {
+          dest.create_temp('column1 integer')
+        }.to change {
+          table_exists?("temp_#{table_name}")
+        }.from(false).to(true)
+      end
+      expect(table_exists?("temp_#{table_name}")).to eq false
+    end
+  end
+end

data/spec/lib/table_copy/pg/field_spec.rb

@@ -0,0 +1,65 @@
+require 'table_copy/pg/field'
+
+describe TableCopy::PG::Field do
+  let(:field) { TableCopy::PG::Field.new(field_attrs) }
+  let(:field_attrs) { { 'column_name' => 'column_name' } }
+
+  context 'for a varchar field' do
+    let(:field_attrs) { {
+      'column_name' => 'column_name',
+      'data_type'   => 'character varying',
+      'character_maximum_length' => '256'
+    } }
+
+    describe '#ddl' do
+      it 'returns a correct segment of ddl' do
+        expect(field.ddl).to eq 'column_name character varying(256)'
+      end
+    end
+
+    describe '#auto_index?' do
+      it 'returns falsey' do
+        expect(field.auto_index?).to be_falsey
+      end
+    end
+  end
+
+  context 'for an integer field' do
+    let(:field_attrs) { {
+      'column_name' => 'column_name',
+      'data_type'   => 'integer'
+    } }
+
+    describe '#ddl' do
+      it 'returns a correct segment of ddl' do
+        expect(field.ddl).to eq 'column_name integer'
+      end
+    end
+
+    describe '#auto_index?' do
+      it 'returns truthy' do
+        expect(field.auto_index?).to be_truthy
+      end
+    end
+  end
+
+  context 'for an array field' do
+    let(:field_attrs) { {
+      'column_name' => 'column_name',
+      'data_type'   => 'ARRAY',
+      'udt_name'    => '_varchar'
+    } }
+
+    describe '#ddl' do
+      it 'returns a correct segment of ddl' do
+        expect(field.ddl).to eq 'column_name character varying(256)[]'
+      end
+    end
+
+    describe '#auto_index?' do
+      it 'returns falsey' do
+        expect(field.auto_index?).to be_falsey
+      end
+    end
+  end
+end

data/spec/lib/table_copy/pg/index_spec.rb

@@ -0,0 +1,24 @@
+require 'table_copy/pg/index'
+
+describe TableCopy::PG::Index do
+  let(:columns) { [ 'column1', 'column2', 'column3' ] }
+  let(:table) { 'table_name' }
+  let(:name) { 'index_name' }
+  let(:index) { TableCopy::PG::Index.new(table, name, columns) }
+
+  describe '#create' do
+    let(:expected) { 'create index on table_name using btree (column1, column2, column3)' }
+
+    it 'returns a correct create index statement' do
+      expect(index.create).to eq expected
+    end
+  end
+
+  describe '#drop' do
+    let(:expected) { 'drop index if exists index_name' }
+
+    it 'returns a correct drop index statement' do
+      expect(index.drop).to eq expected
+    end
+  end
+end

data/spec/lib/table_copy/pg/source_spec.rb

@@ -0,0 +1,120 @@
+require 'table_copy/pg/source'
+require 'yaml'
+
+describe TableCopy::PG::Source do
+  let(:conn) { $pg_conn }
+  let(:table_name) { 'table_name' }
+
+  def with_conn
+    yield conn
+  end
+
+  let(:source) { TableCopy::PG::Source.new(
+    table_name: table_name,
+    conn_method: method(:with_conn)
+  )}
+
+  after do
+    conn.exec("drop table if exists #{table_name}")
+  end
+
+  describe '#to_s' do
+    it 'returns the table name' do
+      expect(source.to_s).to eq table_name
+    end
+  end
+
+  describe '#primary_key' do
+    context 'primary key is defined' do
+      let(:pk) { 'primary_key' }
+      before do
+        conn.exec("create table #{table_name} (#{pk} integer primary key)")
+      end
+
+      it 'returns the name of the primary key' do
+        expect(source.primary_key).to eq pk
+      end
+    end
+
+    context 'pk is not defined' do
+      before do
+        conn.exec("create table #{table_name} (#{pk} integer)")
+      end
+
+      context 'pk inferrence proc is defined' do
+        let(:pk) { "#{table_name}_id" }
+
+        let(:source) { TableCopy::PG::Source.new(
+          table_name: table_name,
+          conn_method: method(:with_conn),
+          infer_pk_proc: Proc.new { |tn| "#{tn}_id" }
+        )}
+
+        it 'returns the name of the primary key' do
+          expect(source.primary_key).to eq pk
+        end
+      end
+
+      context 'pk inferrence proc is not defined' do
+        let(:pk) { "#{table_name}_id" }
+
+        it 'returns "id"' do
+          expect(source.primary_key).to eq 'id'
+        end
+      end
+    end
+  end
+
+  context 'a table exists' do
+    before do
+      conn.exec("create table #{table_name} (column1 integer, column2 varchar(123), column3 varchar(256)[])")
+    end
+
+    describe '#fields_ddl' do
+      it 'returns correct fields ddl' do
+        expect(source.fields_ddl.gsub("\n", '')).to eq 'column1 integer,  column2 character varying(123),  column3 character varying(256)[]'
+      end
+    end
+
+    describe '#fields' do
+      it 'returns an array of field names' do
+        expect(source.fields).to eq [ 'column1', 'column2', 'column3' ]
+      end
+    end
+
+    context 'indexes exist' do
+      before do
+        conn.exec("create index on #{table_name} (column1)")
+      end
+
+      describe '#indexes' do
+        it 'returns an array of indexes' do
+          expect(source.indexes.count).to eq 1
+          index = source.indexes.first
+          expect(index.table).to eq table_name
+          expect(index.columns).to eq [ 'column1' ]
+        end
+      end
+    end
+
+    context 'a row exists' do
+      before do
+        conn.exec("insert into #{table_name} values(1, 'foo', '{bar, baz}')")
+      end
+
+      it 'yields a copying connection' do
+        source.copy_from('column1, column2, column3') do |copy_conn|
+          expect(copy_conn.get_copy_data).to eq "1,foo,\"{bar,baz}\"\n"
+          expect(copy_conn.get_copy_data).to be_nil
+        end
+      end
+    end
+
+  end
+
+  # context 'debugging specs' do
+  #   it 'works' do
+  #     conn.exec("create table #{table_name} (column1 integer,  column2 character varying(123),  column3 character varying(256)[])")
+  #   end
+  # end
+end