wineskins 0.2.2

data/lib/wineskins/record_methods.rb

@@ -0,0 +1,25 @@
+module Wineskins
+
+  module RecordMethods
+  
+    # reads + inserts ten records at a time
+    def transfer_records(table)
+      src_tbl, dst_tbl = table.source_name, table.dest_name
+      rename = table.rename_map(source[src_tbl].columns)
+
+      set_progressbar dst_tbl, source[src_tbl].count
+      
+      source[src_tbl].each_slice(10) do |recs|
+        dest[dst_tbl].multi_insert(
+          recs.map {|rec|
+            remap = Utils.remap_hash(rec, rename)
+            block_given? ? yield(remap) : remap           
+          }
+        )
+        progressbar.inc(10) if progressbar
+      end
+    end
+    
+  end
+  
+end

data/lib/wineskins/runner.rb

@@ -0,0 +1,132 @@
+require 'optparse'
+
+module Wineskins
+
+  class Runner
+  
+    def initialize(argv=nil)
+      @config = Config.new
+      parse_opts argv
+    end
+    
+    def parse_opts(argv)
+      opts = OptionParser.new do |opts|
+        opts.default_argv = argv if argv
+        
+        opts.banner = "Usage: \n"+
+                      "(1) wskins [options] [source-db] [dest-db]\n" +
+                      "(2) wskins [options]"
+        
+        opts.separator nil
+        opts.separator "Options:"
+        
+        opts.on "-c", "--config FILE", "Transfer script, default #{@config.script}" do |file|
+          @config.script = file
+        end
+        
+        opts.on "-r", "--require FILE", "Require ruby file or gem before running" do |file|
+          @config.requires << file
+        end
+        
+        opts.on "--[no-]dry-run", "Don't commit changes" do |bool|
+          @config.dry_run = bool
+        end
+        
+        opts.on "-h", "--help", "Show this message" do
+          puts opts
+          exit
+        end
+        
+        opts.separator nil
+        opts.separator "Summary:"
+        opts.separator wrap( %Q[
+Transfer schema and data from [source-db] to [dest-db], according to 
+instructions in transfer script (by default, #{@config.script}).
+
+In form (1), specify a source and destination database URL (as recognized by 
+Sequel.connect). In form (2), require (-r) a ruby program that connects to 
+databases manually and assigns SOURCE_DB= and DEST_DB=
+
+], 80)
+      end
+      
+      opts.parse!(argv)
+      @config.source = argv.shift
+      @config.dest   = argv.shift
+      
+    end
+    
+    def run
+      require_all
+      set_global_unless_defined 'SOURCE_DB', @config.source_db
+      set_global_unless_defined 'DEST_DB', @config.dest_db
+      validate
+      t = Transfer.new(::SOURCE_DB, ::DEST_DB)
+      eval "t.define {\n" + @config.script_text + "\n}"
+      t.run @config.run_options
+    end
+    
+    # require specified libs
+    def require_all
+      @config.requires.each do |r| require r end
+    end
+
+    def set_global_unless_defined(const, value)
+      unless Object.const_defined?(const)
+        Object.const_set(const, value)
+      end
+      Object.const_get(const)
+    end
+    
+    def validate
+      unless ::SOURCE_DB && ::DEST_DB
+        $stderr.puts wrap( %Q[
+You must specify a source and destination database URL, or manually assign 
+SOURCE_DB= and DEST_DB=
+], 80)
+        exit false
+      end    
+    end
+    
+    private
+    
+    def wrap(text, width) # :doc:
+      text.gsub(/(.{1,#{width}})( +|$\n?)|(.{1,#{width}})/, "\\1\\3\n")
+    end
+    
+  end
+  
+  class Config < Struct.new(:source, :dest, :script, :dry_run)
+  
+    attr_reader :requires
+    def initialize(*args)
+      @requires = []
+      super
+      self.script  ||= "./transfer.rb"
+      self.dry_run ||= false
+    end
+    
+    def source_db
+      return unless self.source
+      @source_db ||=  Sequel.connect(self.source)
+    end
+    
+    def dest_db
+      return unless self.dest
+      @dest_db ||= Sequel.connect(self.dest)
+    end
+    
+    def script_text
+      ::File.read(self.script)
+    end
+    
+    def run_options
+      [:dry_run].inject({}) do |memo, opt|
+        memo[opt] = self.send(opt)
+        memo
+      end
+    end
+    
+  end
+  
+end

data/lib/wineskins/schema_methods.rb

@@ -0,0 +1,100 @@
+module Wineskins
+  
+  module SchemaMethods
+  
+    def transfer_table(table)
+      src_tbl, dst_tbl = table.source_name, table.dest_name
+      rename = table.rename_map(source[src_tbl].columns)
+      alters = table.dest_columns
+      this = self
+      dest.create_table(dst_tbl) do
+        this.source_schema(src_tbl).each do |(fld, spec)|
+          if args = alters[fld]
+            column rename[fld], *args
+          else
+            column_opts = this.schema_to_column_options(spec)
+            column rename[fld], column_opts.delete(:type), column_opts
+          end
+        end
+      end
+    end
+    
+    def transfer_indexes(table)
+      src_tbl, dst_tbl = table.source_name, table.dest_name
+      rename = table.rename_map(source[src_tbl].columns)
+      this = self
+      dest.alter_table(dst_tbl) do
+        this.source_indexes(src_tbl).each do |(name, spec)|
+          index_opts = this.schema_to_index_options(spec)
+          index_cols = spec[:columns].map {|c| rename[c]}
+          add_index index_cols, index_opts
+        end
+      end
+    end
+    
+    def transfer_fk_constraints(table, table_rename={})
+      src_tbl, dst_tbl = table.source_name, table.dest_name
+      rename = table.rename_map(source[src_tbl].columns)
+      this = self
+      dest.alter_table(dst_tbl) do
+        this.source_foreign_key_list(src_tbl).each do |spec|
+          fk_opts = this.schema_to_foreign_key_options(spec)
+          fk_cols = spec[:columns].map {|c| rename[c]}
+          add_foreign_key fk_cols, table_rename[spec[:table]] || spec[:table], fk_opts
+        end
+      end
+    end
+    
+    def source_schema(tbl,opts={})
+      source.schema(tbl,opts)
+    rescue Sequel::Error
+      warn "Source database does not expose schema metadata. " +
+           "You should define schema manually."
+      []
+    end
+    
+    def source_indexes(tbl,opts={})
+      source.indexes(tbl,opts)
+    rescue Sequel::Error
+      warn "Source database does not expose index metadata. " +
+           "You should define indexes manually."
+      {}      
+    end
+    
+    def source_foreign_key_list(tbl,opts={})
+      source.foreign_key_list(tbl,opts)
+    rescue Sequel::Error
+      warn "Source database does not expose foreign key metadata. " +
+           "You should define foreign key constraints manually."
+      []
+    end
+
+    
+    def schema_to_column_options(spec)
+      Utils.remap_hash(
+        Utils.limit_hash(spec, [
+          :primary_key,
+          :default,
+          :allow_null
+        ]),
+        :allow_null => :null
+      )
+    end
+    
+    def schema_to_index_options(spec)
+      Utils.limit_hash spec, [:unique]
+    end
+    
+    def schema_to_foreign_key_options(spec)
+      Utils.limit_hash spec, [
+        :key,
+        :deferrable,
+        :name,
+        :on_delete,
+        :on_update
+      ]
+    end
+    
+  end
+
+end

data/lib/wineskins/transcript.rb

@@ -0,0 +1,24 @@
+module Wineskins
+
+   class Transcript
+  
+    def initialize(io=nil)
+      @io = io || $stdout
+    end
+    
+    # write all sql and errors, stripping the duration from the front 
+    def method_missing(m, msg)
+      write msg.gsub(/\A\([\d\.s]+\)\s+/,'')
+    end
+    
+    def write(msg)
+      if String === @io
+        File.open(@io, 'w+') {|f| f.puts msg}
+      else
+        @io.puts msg
+      end
+    end
+    
+  end
+  
+end

data/lib/wineskins/utils.rb

@@ -0,0 +1,19 @@
+module Wineskins
+
+  module Utils
+    extend self
+   
+    def remap_hash(hash, map)
+      hash.inject({}) do |memo, (k,v)|
+        memo[ map[k] || k ] = v
+        memo
+      end
+    end
+
+    def limit_hash(hash, keys)
+      Hash[ hash.select {|k,v| keys.include?(k)} ]
+    end
+    
+  end
+  
+end

data/lib/wineskins/version.rb

@@ -0,0 +1,3 @@
+module Wineskins
+  VERSION = '0.2.2'
+end

data/lib/wineskins_cli.rb

@@ -0,0 +1,3 @@
+['wineskins', 'wineskins/runner'].each do |rb|
+  require File.expand_path(rb, File.dirname(__FILE__))
+end

data/test/helper.rb

@@ -0,0 +1,128 @@
+gem 'minitest'
+require 'minitest/autorun'
+
+class SequelSpec < MiniTest::Spec
+  def run(*args, &block)
+    TEST_DB.dest.transaction(:rollback => :always){super}
+  end
+end
+
+MiniTest::Spec.register_spec_type(/functional/i, SequelSpec)
+
+
+require 'logger'
+
+class << (TEST_DB = Object.new)
+
+  attr_accessor :source_connect, :dest_connect
+  
+  def source
+    @source ||= Sequel.connect(source_connect)
+  end
+
+  def dest
+    @dest ||= Sequel.connect(dest_connect)
+  end
+  
+  def setup_source(*tables)
+    source.loggers = [::Logger.new('test/log/source.log')]
+    setup source, *tables
+  end
+  
+  def setup_dest(*tables)
+    dest.loggers = [::Logger.new('test/log/dest.log')]
+    setup dest, *tables
+  end
+  
+  def setup(db, *tables)
+    tables.each do |t|
+      TEST_SCHEMA.send(t, db)
+      db[t].multi_insert(
+        TEST_DATA[t]
+      )
+    end
+  end
+  
+end
+
+class << (TEST_SCHEMA = Object.new)
+
+  def tests(db)
+    db.create_table! :tests do
+      primary_key :id
+      column :name, :string
+      column :score, :integer, :index => true
+      column :taken_at, :datetime
+      index :name, :unique => true
+      foreign_key :user_id, :users, :on_delete => :cascade
+      foreign_key :cat_id, :test_categories, :on_update => :cascade
+    end
+  end
+  
+  def users(db)
+    db.create_table! :users do
+      primary_key :uid
+      column :username, :string
+      column :joined_at, :datetime
+    end
+  end
+  
+  def test_categories(db)
+    db.create_table! :test_categories do
+      primary_key :id
+      column :name, :string
+      column :desc, :string
+    end
+  end
+  
+end
+
+TEST_DATA = {
+
+  :tests => [
+    {:id => 1, :name => 'test1', :score => 65, :taken_at => Time.now - rand(60*60*24),
+     :user_id => 11, :cat_id => 1},
+    {:id => 2, :name => 'test2', :score => 21, :taken_at => Time.now - rand(60*60*24),
+     :user_id => 10, :cat_id => 1},
+    {:id => 3, :name => 'test3', :score => 75, :taken_at => Time.now - rand(60*60*24),
+     :user_id =>  9, :cat_id => 1},
+    {:id => 4, :name => 'test4', :score => 84, :taken_at => Time.now - rand(60*60*24),
+     :user_id =>  8, :cat_id => 2},
+    {:id => 5, :name => 'test5', :score => 60, :taken_at => Time.now - rand(60*60*24),
+     :user_id =>  7, :cat_id => 3},
+    {:id => 6, :name => 'test6', :score => 20, :taken_at => Time.now - rand(60*60*24),
+     :user_id =>  1, :cat_id => 1},
+    {:id => 7, :name => 'test7', :score => 66, :taken_at => Time.now - rand(60*60*24),
+     :user_id =>  1, :cat_id => 2},
+    {:id => 8, :name => 'test8', :score => 87, :taken_at => Time.now - rand(60*60*24),
+     :user_id =>  2, :cat_id => 2},
+    {:id => 9, :name => 'test9', :score => 72, :taken_at => Time.now - rand(60*60*24),
+     :user_id =>  4, :cat_id => 3},
+    {:id => 10, :name => 'test10', :score => 33, :taken_at => Time.now - rand(60*60*24),
+     :user_id =>  5, :cat_id => 1},
+    {:id => 11, :name => 'test11', :score => 99, :taken_at => Time.now - rand(60*60*24),
+     :user_id =>  8, :cat_id => 1}  
+  ],
+  
+  :users => [
+    {:uid => 1, :username => "Wilson", :joined_at => Time.now - rand(60*60*24)},
+    {:uid => 2, :username => "Abner",  :joined_at => Time.now - rand(60*60*24)},
+    {:uid => 3, :username => "Penelope", :joined_at => Time.now - rand(60*60*24)},
+    {:uid => 4, :username => "Harrison", :joined_at => Time.now - rand(60*60*24)},
+    {:uid => 5, :username => "Luke", :joined_at => Time.now - rand(60*60*24)},
+    {:uid => 6, :username => "Kramer", :joined_at => Time.now - rand(60*60*24)},
+    {:uid => 7, :username => "Gjert", :joined_at => Time.now - rand(60*60*24)},
+    {:uid => 8, :username => "Yvette", :joined_at => Time.now - rand(60*60*24)},
+    {:uid => 9, :username => "Ruth", :joined_at => Time.now - rand(60*60*24)},
+    {:uid => 10, :username => "Lambert", :joined_at => Time.now - rand(60*60*24)},
+    {:uid => 11, :username => "Percy", :joined_at => Time.now - rand(60*60*24)}  
+  ],
+  
+  :test_categories => [
+    {:id  => 1, :name => 'first', :desc => 'first test'},
+    {:id  => 2, :name => 'second', :desc => 'second test'},
+    {:id  => 3, :name => 'third'}
+  ]
+  
+}
+

data/test/helpers/transfer_assertions.rb

@@ -0,0 +1,102 @@
+module TransferAssertions
+  
+  # I wish we could compare :type as well, but this seems somewhat adapter-
+  # dependent. Probably :default is adapter-dependent too.
+  #
+  def assert_columns_match(table)
+    match_keys = [:allow_null, :primary_key, :default]
+    exp = source.schema(table).map {|name, col| 
+            [name, Wineskins::Utils.limit_hash(col, match_keys)]
+          }
+    act = dest.schema(table).map {|name, col| 
+            [name, Wineskins::Utils.limit_hash(col, match_keys)]
+          }
+    assert_equal exp.length, act.length,
+      "Expected #{exp.length} columns in table #{table}, got #{act.length}"
+    exp.each do |col|
+      assert_includes act, col,
+        "Unexpected column options for #{col[0]} in table #{table}"
+    end
+  end
+  
+  def assert_column_matches(table, col, specs=nil)
+    src_col, dst_col = Array(col)
+    dst_col ||= src_col
+    exp = if specs 
+            [src_col, specs]
+          else
+            source.schema(table).find {|name,col| name == src_col}
+          end
+    act = dest.schema(table).find {|name,col| name == dst_col}
+    refute_nil act, "Expected column #{dst_col} not found in table #{table}"
+    
+    match_keys = [:allow_null, :primary_key, :default]
+    exp = [exp[0], Wineskins::Utils.limit_hash( exp[1], match_keys )]
+    act = [act[0], Wineskins::Utils.limit_hash( act[1], match_keys )]
+    assert_equal exp[1], act[1],
+      "Unexpected column options for #{dst_col} in table #{table}"
+  end
+  
+  def assert_indexes_match(table)
+    exp, act = source.indexes(table), dest.indexes(table)
+    assert_equal exp.length, act.length, 
+      "Expected #{exp.length} indexes for table #{table}, got #{act.length}"
+    exp.values.each do |idx|
+      assert_includes act.values, idx,
+        "Unexpected index options in table #{table}"
+    end
+  end
+  
+  def assert_index_matches(table, cols, rename={})
+    src_cols = Array(cols)
+    dst_cols = src_cols.map {|c| rename[c] || c}
+    exp = source.indexes(table).find {|name, specs| 
+      specs[:columns] == src_cols
+    }
+    act = dest.indexes(table).find {|name, specs| 
+      specs[:columns] == dst_cols
+    }
+    refute_nil act, "Expected index #{dst_cols.inspect} not found in table #{table}"
+    
+    match_keys = [:unique]
+    exp = Wineskins::Utils.limit_hash(exp[1], match_keys)
+    act = Wineskins::Utils.limit_hash(act[1], match_keys)
+    assert_equal exp, act,
+      "Unexpected index options for #{dst_cols.inspect} in table #{table}"
+  end
+  
+  def assert_fk_match(table)
+    match_keys = [:columns, :table, :key, :on_delete, :on_update]
+    exp = source.foreign_key_list(table).map {|fk| 
+            Wineskins::Utils.limit_hash(fk, match_keys) 
+          }
+    act = dest.foreign_key_list(table).map {|fk| 
+            Wineskins::Utils.limit_hash(fk, match_keys) 
+          }
+    assert_equal exp.length, act.length,
+      "Expected #{exp.length} foreign key constraints for table #{table}, got #{act.length}"
+    exp.each do |fk|
+      assert_includes act, fk, 
+        "Unexpected foreign key options in table #{table}"
+    end
+  end
+  
+  def assert_fk_matches(table, cols, rename={})
+    src_cols = Array(cols)
+    dst_cols = src_cols.map {|c| rename[c] || c}
+    exp = source.foreign_key_list(table).find {|specs| 
+      specs[:columns] == src_cols
+    }
+    act = dest.foreign_key_list(table).find {|specs| 
+      specs[:columns] == dst_cols
+    }
+    refute_nil act, "Expected foreign key #{dst_cols.inspect} not found in table #{table}"
+    
+    match_keys = [:table, :key, :on_delete, :on_update]
+    exp = Wineskins::Utils.limit_hash(exp, match_keys)
+    act = Wineskins::Utils.limit_hash(act, match_keys)
+    assert_equal exp, act,
+      "Unexpected foreign key options for #{dst_cols.inspect} in table #{table}"  
+  end
+  
+end