table_copy 0.0.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 82e2e7bc03e341d5b10e172c080499d174abd238
+  data.tar.gz: ccf3b5eebc929cefe8b9eeec212d7869b3cacb2c
+SHA512:
+  metadata.gz: 9be2278b5259705cbfb905422385a334145eeb5325d8f97b33eb253343e3838014d081e3f86417c7456c519364ad29e19ea40e3cedece2fe84550ca2514c9567
+  data.tar.gz: 4191cd39eda48dd99414e3f300a962d6de22e0ad756aaa9f7da8755ac5d0a4d42d14d185d4cf717a40e80524b254f8a80bbfacd4b4c842ba02dbf885a4e698cf

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in table_copy.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Tyler Hartland
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,45 @@
+# TableCopy
+
+Move and update data on a table by table basis between two databases. Currently only supports Postgres in a limited fashion.
+
+This gem could be made more flexible with a bit of work, but for now is pretty limited to my specific purposes.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'table_copy'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install table_copy
+
+## Usage
+
+Run ```table_copy --init``` for an example initializer. Then, access each table link by ```TableCopy.links['link_name']```. You can call ```link.update; link.droppy; link.diffy```
+
+Update will attempt to use a sequence field to look for changes. If that field is not available, it will run a diffy(update) operation.
+
+Diffy(update) will copy the source table to a temp table, diff it with the destination table, and upsert any changes to the destination.
+
+Diffy will perform a diffy(update) and will also diff ids in the destination table against the temp table to find deletions.
+
+Droppy will drop the destination table and rebuild/populate it.
+
+### *Very* rough benchmarks:
+- Copy 1M rows ~15 sec
+- Index 1M rows ~2 sec per numeric field, ~40 sec per char field
+- Diff 1M rows ~40 sec
+- Upsert 100k rows into 1M row table ~60 sec
+
+## Contributing
+
+1. Fork it ( https://github.com/th7/table_copy/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/table_copy

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+bin_path = File.expand_path(File.dirname(__FILE__))
+system('mkdir config') unless Dir.exists?('config')
+system('mkdir config/initializers') unless Dir.exists?('config/initializers')
+if system("cp #{bin_path}/../config/initializers/table_copy.rb.example config/initializers")
+  puts "Example initializer copied to /config/initializers/table_copy.rb.example"
+else
+  puts 'Failed to copy example config.'
+end
+

data/config/database.yml

@@ -0,0 +1 @@
+:dbname: table_copy_test

data/config/initializers/table_copy.rb.example

@@ -0,0 +1,62 @@
+require 'yaml'
+require 'pg'
+require 'table_copy'
+require 'table_copy/pg'
+
+TABLES = {
+  'table_one'   => { sequence_field: 'updated_at' },
+  'table_two'   => { skips_fields: [ 'field_to_skip' ] },
+  'table_three' => { sequence_field: 'table_three_id' }, #insert only
+}
+
+# TableCopy requires you to specify methods which yield a database connection
+# single connection example
+source_config = YAML.load_file('config/db1.yml')ENV['ENV']
+source_conn   = PG::Connection.open(source_config)
+
+class SourceDB
+  def self.with_conn
+    yield source_conn # or use a connection pool!
+  end
+end
+
+# Active Record connection pool example
+class DestinationDB < ActiveRecord::Base
+  self.abstract_class = true
+
+  def self.with_conn
+    self.connection_pool.with_connection do |ar_conn|
+      yield ar_conn.raw_connection
+    end
+  end
+end
+
+TableCopy.logger = Logger.new('log/table_copy.log') unless MyEnv.is.development?
+
+# if explicitly asking the DB for the PK fails, a proc can be used instead
+infer_pk_proc = Proc.new { |table_name| table_name + '_id' }
+
+# or maybe...
+# infer_pk_proc = Proc.new { 'every_table_uses_this_id' }
+
+# config requires database queries -- this block defers until it is actually needed
+TableCopy.deferred_config do
+  TABLES.each do |table_name, opts|
+    source = TableCopy::PG::Source.new(
+      table_name:    table_name,
+      conn_method:   SourceDB.method(:with_conn),
+      infer_pk_proc: infer_pk_proc
+    )
+
+    destination = TableCopy::PG::Destination.new(
+      table_name:     table_name,
+      primary_key:    source.primary_key,
+      sequence_field: opts[:sequence_field],
+      conn_method:    DestinationDB.method(:with_conn),
+      indexes:        source.indexes,
+      fields:         source.fields - (opts[:skips_fields] || [])
+    )
+
+    TableCopy.add_link(table_name, source, destination)
+  end
+end

data/lib/table_copy/copier.rb

@@ -0,0 +1,98 @@
+require 'pg'
+
+module TableCopy
+  class Copier
+    attr_reader :source_table, :destination_table
+
+    def initialize(source_table, destination_table)
+      @source_table      = source_table
+      @destination_table = destination_table
+    end
+
+    def update
+      if destination_table.none?
+        droppy
+      elsif (max_sequence = destination_table.max_sequence)
+        update_data(max_sequence)
+      else
+        diffy_update
+      end
+    rescue ::PG::UndefinedTable => e
+      ([e.inspect] + e.backtrace).each { |l| logger.warn(l) }
+      create_table
+      retry
+    rescue ::PG::UndefinedColumn => e
+      ([e.inspect] + e.backtrace).each { |l| logger.warn(l) }
+      droppy
+    end
+
+    def droppy
+      logger.info { "Droppy #{destination_table.table_name}" }
+      destination_table.transaction do
+        destination_table.drop(cascade: true)
+        create_table
+        moved_count = destination_table.copy_data_from(source_table)
+        logger.info { "#{moved_count} rows moved to #{destination_table.table_name}" }
+        destination_table.create_indexes
+        logger.info { "Completed #{source_table.indexes.count} indexes on #{destination_table.table_name}." }
+      end
+    end
+
+    def find_deletes
+      logger.info { "Find deletes #{destination_table.table_name}" }
+      destination_table.transaction do
+        destination_table.create_temp(source_table.fields_ddl)
+        moved_count = destination_table.copy_data_from(source_table, temp: true, pk_only: true)
+        logger.info { "#{moved_count} rows moved to temp_#{destination_table.table_name}" }
+        destination_table.delete_not_in_temp
+        logger.info { "Deletetions from #{destination_table.table_name} complete." }
+      end
+    end
+
+    def diffy
+      logger.info { "Diffy #{destination_table.table_name}" }
+      destination_table.transaction do
+        destination_table.create_temp(source_table.fields_ddl)
+        moved_count = destination_table.copy_data_from(source_table, temp: true)
+        logger.info { "#{moved_count} rows moved to temp_#{destination_table.table_name}" }
+        destination_table.copy_from_temp
+        logger.info { "Upsert to #{destination_table.table_name} complete" }
+        destination_table.delete_not_in_temp
+        logger.info { "Deletetions from #{destination_table.table_name} complete." }
+      end
+    end
+
+    private
+
+    def diffy_update
+      logger.info "Diffy Update #{destination_table.table_name}"
+      destination_table.transaction do
+        destination_table.create_temp(source_table.fields_ddl)
+        moved_count = destination_table.copy_data_from(source_table, temp: true)
+        logger.info "#{moved_count} rows moved to temp_#{destination_table.table_name}"
+        destination_table.copy_from_temp
+        logger.info "Upsert to #{destination_table.table_name} complete."
+      end
+    end
+
+    def update_data(max_sequence)
+      logger.info "Update #{destination_table.table_name}"
+      destination_table.transaction do
+        destination_table.create_temp(source_table.fields_ddl)
+        moved_count = destination_table.copy_data_from(source_table, temp: true, update: max_sequence)
+        logger.info "#{moved_count} rows moved to temp_#{destination_table.table_name}"
+        destination_table.copy_from_temp(except: nil)
+        logger.info "Upsert to #{destination_table.table_name} complete."
+      end
+    end
+
+    def create_table
+      logger.info { "Creating table #{destination_table.table_name}" }
+      destination_table.create(source_table.fields_ddl)
+    end
+
+    def logger
+      TableCopy.logger
+    end
+  end
+end

data/lib/table_copy/pg/destination.rb

@@ -0,0 +1,162 @@
+module TableCopy
+  module PG
+    class Destination
+      attr_reader :table_name, :conn_method, :indexes, :fields, :primary_key, :sequence_field
+
+      def initialize(args)
+        @table_name     = args[:table_name]
+        @primary_key    = args[:primary_key]
+        @sequence_field = args[:sequence_field]
+        @conn_method    = args[:conn_method]
+        @indexes        = args[:indexes] || []
+        @fields         = args[:fields]
+      end
+
+      def transaction
+        with_conn do |conn|
+          begin
+            conn.exec('begin')
+            yield
+            conn.exec('commit')
+          rescue Exception => e
+            conn.exec('rollback')
+            raise e
+          end
+        end
+      end
+
+      def create(fields_ddl)
+        with_conn do |conn|
+          conn.exec("create table #{table_name} (#{fields_ddl})")
+        end
+      end
+
+      def drop(opts={})
+        cascade = ' cascade' if opts[:cascade]
+        with_conn do |conn|
+          conn.exec("#{drop_sql}#{cascade}")
+        end
+      end
+
+      def create_indexes
+        indexes.each do |index|
+          create_ddl = index.class.new(table_name, index.name, index.columns).create
+          with_conn do |conn|
+            conn.exec(create_ddl)
+          end
+        end
+      end
+
+      def to_s
+        table_name
+      end
+
+      def max_sequence
+        return unless sequence_field
+        with_conn do |conn|
+          row = conn.exec(max_sequence_sql).first
+          row['max'] if row
+        end
+      end
+
+      def create_temp(fields_ddl)
+        with_conn do |conn|
+          conn.exec("create temp table temp_#{table_name} (#{fields_ddl}) on commit drop")
+        end
+      end
+
+      def none?
+        with_conn do |conn|
+          conn.exec("select count(*) from #{table_name}").first['count'] == '0'
+        end
+      end
+
+      def copy_data_from(source_table, temp: nil, pk_only: false, update: false)
+        temp = 'temp_' if temp
+        fl = pk_only ? primary_key : fields_list
+        where = "where #{sequence_field} > '#{update}'" if update && sequence_field
+        count = 0
+        source_table.copy_from(fl, where) do |source_conn|
+          with_conn do |conn|
+            conn.copy_data("COPY #{temp}#{table_name} (#{fl}) FROM STDOUT CSV") do
+              while row = source_conn.get_copy_data
+                count += 1
+                conn.put_copy_data(row)
+              end
+            end
+          end
+        end
+        count
+      end
+
+      def copy_from_temp(except: except_statement)
+        with_conn do |conn|
+          conn.exec(upsert_sql(except))
+        end
+      end
+
+      def delete_not_in_temp
+        with_conn do |conn|
+          conn.exec("delete from #{table_name} where #{primary_key} in (select #{primary_key} from #{table_name} except select #{primary_key} from temp_#{table_name})")
+        end
+      end
+
+      private
+
+      def fields_list
+        @fields_list ||= fields.join(', ')
+      end
+
+      def with_conn(&block)
+        conn_method.call(&block)
+      end
+
+      attr_reader :primary_key
+
+      def drop_sql
+        @drop_sql ||= "drop table if exists #{table_name}"
+      end
+
+      def max_sequence_sql
+        @max_sequence_sql ||= "select max(#{sequence_field}) from #{table_name}"
+      end
+
+      def upsert_sql(except=except_statement)
+        "with new_values as (
+          select #{fields_list} from temp_#{table_name}
+          #{except}
+        )
+        ,upsert as (
+          UPDATE #{table_name}
+          SET #{set_statement(fields)}
+          FROM new_values as nv
+          WHERE #{table_name}.#{primary_key} = nv.#{primary_key}
+          RETURNING #{return_statement(fields)}
+        )
+
+        INSERT INTO #{table_name} (#{fields_list})
+               SELECT *
+               FROM new_values as nv
+               WHERE NOT EXISTS (SELECT 1
+                                 FROM #{table_name}
+                                 WHERE #{table_name}.#{primary_key} = nv.#{primary_key});"
+      end
+
+      def except_statement
+        @except_statement ||= "except select #{fields_list} from #{table_name}"
+      end
+
+      def set_statement(keys)
+        keys.map.with_index(1) do |key, i|
+          "#{key}=nv.#{key}"
+        end.join(',')
+      end
+
+      def return_statement(keys)
+        keys.map.with_index(1) do |key, i|
+          "nv.#{key}"
+        end.join(',')
+      end
+    end
+  end
+end

data/lib/table_copy/pg/field.rb

@@ -0,0 +1,42 @@
+module TableCopy
+  module PG
+    class Field
+      attr_reader :name, :type_name, :data_limit
+
+      def initialize(attrs)
+        @name = attrs['column_name']
+        data_type = attrs['data_type']
+
+        if data_type =~ /character/
+          @data_limit = attrs['character_maximum_length']
+        end
+
+        if data_type == 'ARRAY' && attrs['udt_name'] == '_varchar'
+          @type_name = 'character varying'
+          @data_limit = '256'
+          @array_ddl = '[]'
+        end
+
+        @type_name ||= data_type
+      end
+
+      def ddl
+        @ddl ||= "#{name} #{type_name}#{data_limit_ddl}#{array_ddl}"
+      end
+
+      def auto_index?
+        @type_name =~ /int|timestamp|bool/
+      end
+
+      private
+
+      def data_limit_ddl
+        "(#{data_limit})" if @data_limit
+      end
+
+      def array_ddl
+        @array_ddl
+      end
+    end
+  end
+end

data/lib/table_copy/pg/index.rb

@@ -0,0 +1,21 @@
+module TableCopy
+  module PG
+    class Index
+      attr_reader :table, :name, :columns
+
+      def initialize(table, name, columns)
+        @table   = table
+        @name    = name
+        @columns = columns
+      end
+
+      def create
+        @create ||= "create index on #{table} using btree (#{columns.join(', ')})"
+      end
+
+      def drop
+        @drop ||= "drop index if exists #{name}"
+      end
+    end
+  end
+end

data/lib/table_copy/pg/source.rb

@@ -0,0 +1,139 @@
+require 'table_copy/pg/field'
+require 'table_copy/pg/index'
+
+module TableCopy
+  module PG
+    class Source
+      attr_reader :table_name, :conn_method, :infer_pk_proc
+
+      def initialize(args)
+        @table_name    = args[:table_name]
+        @conn_method   = args[:conn_method]
+        @infer_pk_proc = args[:infer_pk_proc]
+      end
+
+      def to_s
+        table_name
+      end
+
+      def primary_key
+        @primary_key ||= get_primary_key
+      end
+
+      def fields_ddl
+        @fields_ddl ||= fields_objects.map(&:ddl).join(",\n  ")
+      end
+
+      def indexes
+        @indexes ||= viable_index_columns.map { |name, columns| TableCopy::PG::Index.new(table_name, name, columns) }
+      end
+
+      def copy_from(fields_list_arg, where=nil)
+        with_conn do |conn|
+          conn.copy_data("copy (select #{fields_list_arg} from #{table_name} #{where}) to stdout csv")  do
+            yield conn
+          end
+        end
+      end
+
+      def fields
+        @field_names ||= fields_objects.map(&:name)
+      end
+
+      private
+
+      def with_conn(&block)
+        conn_method.call(&block)
+      end
+
+      def fields_objects
+        @fields_objects ||= with_conn do |conn|
+          conn.exec(fields_sql).map { |r| TableCopy::PG::Field.new(r) }
+        end
+      end
+
+      def viable_index_columns
+        @viable_index_columns ||= index_columns.select do |name, columns|
+          (columns - fields).empty?
+        end
+      end
+
+      def index_columns
+        @index_columns ||= raw_indexes.inject({}) do |indexes, ri|
+          index_name = ri['index_name']
+          indexes[index_name] ||= []
+          indexes[index_name] << ri['column_name']
+          indexes
+        end
+      end
+
+      def raw_indexes
+        @raw_indexes || with_conn do |conn|
+          @raw_indexes = conn.exec(indexes_sql)
+        end
+      end
+
+      def indexes_sql
+        <<-SQL
+          select
+              i.relname as index_name,
+              a.attname as column_name
+          from
+              pg_class t,
+              pg_class i,
+              pg_index ix,
+              pg_attribute a
+          where
+              t.oid = ix.indrelid
+              and i.oid = ix.indexrelid
+              and a.attrelid = t.oid
+              and a.attnum = ANY(ix.indkey)
+              and t.relkind = 'r'
+              and t.relname = '#{table_name}'
+          order by
+              t.relname,
+              i.relname;
+        SQL
+      end
+
+      def fields_sql
+        <<-SQL
+          SELECT *
+          FROM information_schema.columns
+          WHERE table_schema='public' AND table_name='#{table_name}'
+        SQL
+      end
+
+      def get_primary_key
+        with_conn do |conn|
+          rows = conn.exec(primary_key_sql)
+          if (row = rows.first) && row['attname']
+            row['attname']
+          elsif infer_pk_proc
+            inferred_pk = infer_pk_proc.call(table_name)
+            TableCopy.logger.warn "No explicit PK found for #{table_name}. Falling back to #{inferred_pk}."
+            inferred_pk
+          else
+            TableCopy.logger.warn "No explicit PK found for #{table_name}. Falling back to \"id\"."
+            'id'
+          end
+        end
+      end
+
+      def primary_key_sql
+        <<-SQL
+          SELECT
+          pg_attribute.attname,
+          format_type(pg_attribute.atttypid, pg_attribute.atttypmod)
+        FROM pg_index, pg_class, pg_attribute
+        WHERE
+          pg_class.oid = '#{table_name}'::regclass AND
+          indrelid = pg_class.oid AND
+          pg_attribute.attrelid = pg_class.oid AND
+          pg_attribute.attnum = any(pg_index.indkey)
+          AND indisprimary
+        SQL
+      end
+    end
+  end
+end

data/lib/table_copy/pg.rb

@@ -0,0 +1,9 @@
+require 'table_copy/pg/destination'
+require 'table_copy/pg/field'
+require 'table_copy/pg/index'
+require 'table_copy/pg/source'
+
+module TableCopy
+  module PG
+  end
+end

data/lib/table_copy/version.rb

@@ -0,0 +1,3 @@
+module TableCopy
+  VERSION = "0.0.5"
+end