rigrate 0.0.1

data/rigrate.gemspec

@@ -0,0 +1,21 @@
+Gem::Specification.new do |s|
+  s.name        = 'rigrate'
+  s.version     = '0.0.1'
+  s.executables << 'rigrate'
+  s.date        = '2016-04-01'
+  s.summary     = 'Ruby Migrate! data migrate tool in ruby.'
+  s.description = 'a data migrate tool between diffenect data sources write by ruby.'
+  s.authors     = ['jiyaping']
+  s.email       = 'jiyaping0802@gmail.com'
+  s.homepage    = 'http://rubygems.org/gems/rigrate'
+  s.license     = 'MIT'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- test/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'ruby-oci8', '~> 2.2'
+  s.add_dependency 'sqlite3', '~> 1.3'
+  s.add_dependency "mysql", "~> 2.9"
+end

data/test/test_datasource.rb

@@ -0,0 +1,66 @@
+require File.expand_path('../test_helper', __FILE__)
+
+class DataSourceTest < TestHelper
+  def setup
+    @ds = DataSource.new("sqlite://memory")
+
+    setup_sql =<<SQL
+    create table users (
+      id integer primary key not null,
+      name varchar,
+      age integer,
+      birthday datetime default current_date
+    );
+
+    insert into users(id, name, age) values(1, 'jyp1', 23);
+    insert into users(id, name, age) values(2, 'jyp2', 24);
+    insert into users(id, name, age) values(3, 'jyp3', 25);
+    insert into users(id, name, age) values(4, 'jyp4', 26);
+
+    insert into users(id, name, age) values(5, 'jyp5', 23);
+    insert into users(id, name, age) values(6, 'jyp6', 24);
+    insert into users(id, name, age) values(7, 'jyp7', 25);
+    insert into users(id, name, age) values(8, 'jyp8', 26);    
+SQL
+
+    # setup data
+    @ds.dbh.execute_batch(setup_sql)
+  end
+
+  def test_new_data_source
+    assert @ds
+  end
+
+  def test_new_data_source_2
+    assert DataSource.new("sqlite://#{File.join(Dir.tmpdir, 'tmptest.sqlite3')}")
+  end
+
+  def test_build_sql
+    assert_equal "select * from user", @ds.send(:build_sql, 'user')
+    assert_equal "select name,age from user", @ds.send(:build_sql, 'user', 'name', 'age')
+  end
+
+  def test_sql_block_given
+    result = @ds.sql('select * from users') do |row|
+      row
+    end
+
+    assert_equal 8, result.size
+  end
+
+  def test_sql
+    result = @ds.sql('select * from users')
+
+    assert_kind_of ResultSet, result
+    assert_equal 8, result.size
+  end
+
+  def test_method_missing
+    result = @ds.users(:name, :age) do |row|
+      row
+    end
+
+    assert_kind_of ResultSet, result
+    assert_equal 8, result.size
+  end
+end

data/test/test_driver.rb

@@ -0,0 +1,45 @@
+require File.expand_path('../test_helper', __FILE__)
+
+class DriverTest < TestHelper
+  def setup
+    @inst_str = "test test"
+    @driver = Driver.new
+    @driver.db = @inst_str
+  end
+
+  def test_driver_not_null
+    assert @driver
+  end
+
+  def test_extract_tbl_from_sql
+    tbl_name = "user"
+    tbl_sql = "select * from user"
+
+    assert_equal tbl_name, @driver.extract_tbl_from_sql(tbl_sql)
+  end
+
+  def test_extract_tbl_from_sql2
+    tbl_name = "user"
+    tbl_sql = "select * from   user where xxx = 'from'"
+
+    assert_equal tbl_name, @driver.extract_tbl_from_sql(tbl_sql)
+  end
+
+  def test_extract_tbl_from_sql3
+    tbl_name = "user"
+    tbl_sql = "select * from   user from test" # invalid sql form
+
+    assert_equal tbl_name, @driver.extract_tbl_from_sql(tbl_sql)
+  end
+
+  def test_method_missing
+    execpt_size = @inst_str.length
+    assert_equal execpt_size, @driver.length
+  end
+
+  def test_extract_conn_param
+    result = @driver.send(:extract_conn_param, URI.parse("mysql://localhost:3306"))
+    assert_equal result.size, 3
+    assert_equal result['db_type'], 'mysql'
+  end
+end

data/test/test_driver_mysql.rb

@@ -0,0 +1,87 @@
+require File.expand_path('../test_helper', __FILE__)
+
+class Mysqltest < TestHelper
+  def setup
+    @db = Mysql.new("mysql://root:20080802@127.0.0.1/test")
+    stm1 = @db.prepare(get_seed_data('tbl1'))
+    stm1.execute
+    stm1 = @db.prepare("insert into tbl1 values(?,?,?,?,?)")
+    10.times do |i|
+      args = [i, "jyp#{i}", 20+i, 'tbl1', "2001-01-#{1+i}"]
+      stm1.execute(*args)
+    end
+
+    stm1 = @db.prepare(get_seed_data('tbl2'))
+    stm1.execute
+    stm1 = @db.prepare("insert into tbl2 values(?,?,?,?,?)")
+    5.times do |i|
+      args = [i, "jyp#{i}", 20+i, 'tbl2', "2001-02-#{1+i}"]
+      stm1.execute(*args)
+    end
+  end
+
+  def test_new_mysql
+    assert @db
+  end
+
+  def test_transaction?
+    assert_respond_to @db, :transaction_active?
+  end
+
+  def test_blob_read_and_write
+    stm = @db.prepare("create table if not exists test_blob(id integer, pic blob)")
+    stm.execute
+
+    # insert blob
+    test_str = "xxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    stm_i = @db.prepare("insert into test_blob values(?,?)")
+    assert stm_i.execute(1, test_str)
+
+    # read blob
+    rs = @db.select("select * from test_blob")
+    assert test_str, rs.rows.first[1]
+
+    # clean db
+    stm = @db.prepare("drop table test_blob")
+    stm.execute
+  end
+
+  def test_mysql_migration
+    str =<<SCRIPT
+    ds :oa, "mysql://root:20080802@127.0.0.1/test"
+    ds :oa_bak,"mysql://root:20080802@127.0.0.1/test"
+
+    from oa.tbl1 to oa_bak.tbl2
+SCRIPT
+   
+    parser = Parser.new
+    parser.lex(str)
+    parser.parsing
+    rs = @db.select("select * from tbl2")
+    assert_equal 10, rs.size
+    assert_equal 'tbl1', rs.rows.first[3]
+  end
+
+  private
+
+  def get_seed_data(tbl_name)
+    prepare_a_data =<<SQL
+    create table if not exists #{tbl_name}(
+      id int,
+      name varchar(255),
+      age int,
+      flag varchar(255),
+      birth date
+    );
+SQL
+    
+    prepare_a_data
+  end
+
+  def teardown
+    stm = @db.prepare("drop table tbl1")
+    stm.execute
+    stm = @db.prepare("drop table tbl2")
+    stm.execute
+  end
+end

data/test/test_driver_oracle.rb

@@ -0,0 +1,109 @@
+require File.expand_path('../test_helper', __FILE__)
+
+class OracleTest < TestHelper
+  def setup
+    @db = Oracle.new("oracle://scott:1234567@jtest")
+    @db.db.autocommit = true
+
+    # test for blob
+    @db.exec("create table test_blob(id number(11), pic blob)")
+    @db.exec("create table test_blob2(id number(11), pic blob)")
+
+    # prepare one row
+    @test_str = "xxxxxxxxxx"
+    b_var = OCI8::BLOB.new(@db.db, @test_str)
+    stm = @db.parse("insert into test_blob values (:1,:2)")
+    stm.exec(1, b_var)
+    stm.exec(2, b_var)
+
+    # test for data migration
+    @db.exec("create table user1(id number(11) primary key, name varchar(255), age number(4), flag varchar(255))")
+    stm = @db.parse("insert into user1 values(:1, :2, :3, :4)")
+    5.times do |i|
+      stm.exec(i, "user1+#{i}", 20+i, "user1")
+    end
+    @db.exec("create table user2(id number(11) primary key, name varchar(255), age number(4), flag varchar(255))")
+    stm = @db.parse("insert into user2 values(:1, :2, :3, :4)")
+    5.times do |i|
+      stm.exec(i+2, "user2+#{i}", 30+i, "user2")
+    end
+  end
+
+  def test_new_connection
+    assert @db
+  end
+
+  def test_respond_transaction
+    assert_respond_to @db, :transaction_active?
+  end
+
+  def test_convert_question_mark_to_sym
+    e_sql_1 = "select * from users where id = ?"
+    a_sql_1 = "select * from users where id = :1"
+    assert_equal a_sql_1, @db.convert_question_mark_to_symbol(e_sql_1, ['a'].size)
+
+    e_sql_2 = "select * from users"
+    a_sql_2 = "select * from users"
+    assert_equal a_sql_2, @db.convert_question_mark_to_symbol(e_sql_2, [].size)
+
+    e_sql_3 = "select * from users where id = ? and name = ?"
+    a_sql_3 = "select * from users where id = :1 and name = :2"
+    assert_equal a_sql_3, @db.convert_question_mark_to_symbol(e_sql_3, ['a', 'b'].size)
+  end
+
+  def test_blob_read_and_write
+    stm = @db.parse("select * from test_blob")
+    stm.exec
+    row = stm.fetch
+    assert_equal @test_str, row[1].read
+
+    stm.exec
+    row = stm.fetch
+    rb_row = @db.to_rb_row(row)
+    assert_equal [1, @test_str], rb_row
+  end
+
+  def test_select_rs
+    rs = @db.select("select * from test_blob")
+    assert_equal 2, rs.size
+    assert_equal @test_str, rs.rows.first[1]
+  end
+
+  def test_oracle_migration
+    str =<<SCRIPT
+    ds :oa, "oracle://scott:1234567@jtest"
+    ds :hr, "oracle://scott:1234567@jtest"
+    
+    from oa.user1 to hr.user2
+SCRIPT
+
+    parser = Parser.new
+    parser.lex(str).parsing
+    @rs = @db.select("select * from user2")
+    assert_equal 5, @rs.size
+    assert_equal 'user1', @rs.rows.first[3]
+  end
+
+  def test_blob_migration
+    str =<<SCRIPT
+    ds :oa, "oracle://scott:1234567@jtest"
+    ds :hr, "oracle://scott:1234567@jtest"
+
+    from oa.test_blob to hr.test_blob2
+SCRIPT
+
+    parser = Parser.new
+    parser.lex(str).parsing
+    @rs = @db.select("select * from test_blob2")
+    assert_equal 2, @rs.size
+    assert_equal @test_str, @rs.rows.first[1]
+  end
+
+  def teardown
+    # clean environment
+     @db.exec("drop table test_blob")
+     @db.exec("drop table test_blob2")
+     @db.exec('drop table user1')
+     @db.exec('drop table user2')
+  end
+end

data/test/test_driver_sqlite.rb

@@ -0,0 +1,85 @@
+require File.expand_path('../test_helper', __FILE__)
+
+class SqliteTest < TestHelper
+  def setup
+    @tbl_name = 'users'
+
+    @db =  Sqlite.new
+    setup_sql =<<SQL
+    create table users (
+      id integer primary key not null,
+      name varchar,
+      age integer,
+      birthday datetime default current_date
+    );
+
+    insert into users(id, name, age) values(1, 'jyp1', 23);
+    insert into users(id, name, age) values(2, 'jyp2', 24);
+    insert into users(id, name, age) values(3, 'jyp3', 25);
+    insert into users(id, name, age) values(4, 'jyp4', 26);
+SQL
+
+    # setup data
+    @db.execute_batch(setup_sql)
+  end
+
+  def test_new_db
+    assert @db
+  end
+
+  def test_table_users_exists?
+    sql = "select name from sqlite_master\
+     where type='table' and name='#{@tbl_name}'"
+
+    result = @db.execute sql
+    if result.size > 0
+      q_tbl_name = result.first.first
+    end
+
+    assert_equal q_tbl_name, @tbl_name
+  end
+
+  def test_primary_key_returned_value
+    result = @db.primary_key('users')
+
+    assert Array === result
+  end
+
+  def test_get_users_primary_is_id
+    result = @db.primary_key('users')
+
+    assert_equal ['id'], result
+  end
+
+  def test_insert_values
+    sql = "insert into users(id, name) values (1000, 'test')"
+    assert @db.insert(sql)
+    sql = "insert into users(id, name) values (?, ?)"
+    args = [10001, 'testtest']
+    assert @db.insert(sql, args)
+  end
+
+  def test_inherit_extract_tblname
+    sql = "select * from users"
+
+    assert_equal "users", @db.extract_tbl_from_sql(sql)
+  end
+
+  def test_select_is_success
+    sql = "select id, name from users"
+    assert @db.select sql
+  end
+
+  def test_delete
+    assert @db.delete("delete from users where id=?", [1])
+  end
+
+  def test_extract_db_path
+    assert ":memory:", @db.extract_db_path("sqlite://memory")
+    assert "/home/test.db", @db.extract_db_path("sqlite://home/test.db")
+  end
+
+  def teardown
+    #@db.close unless @db.closed?
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,9 @@
+gem "minitest"
+require 'minitest/autorun'
+require 'fileutils'
+require 'tmpdir'
+require 'rigrate'
+
+class TestHelper < MiniTest::Test
+  include Rigrate
+end

data/test/test_migration.rb

@@ -0,0 +1,108 @@
+require File.expand_path('../test_helper', __FILE__)
+
+class MigrationTest < TestHelper
+  def setup
+    @obj = Object.new.extend(Migration)
+
+    # seed data
+    @obj.self_eval("ds :oa, 'sqlite://memory'")
+    @obj.oa.dbh.execute_batch(get_seed_sql('oa'))
+
+    @obj.self_eval("ds :hr, 'sqlite://memory'")
+    @obj.hr.dbh.execute_batch(get_seed_sql('hr'))    
+  end
+
+  def test_self_eval
+    assert_equal 3, @obj.self_eval('1+2')
+  end
+
+  def test_self_eval2
+    assert_kind_of ResultSet, @obj.self_eval("oa.dbh.select('select * from users')")
+  end
+
+  def test_data_source
+    @obj.ds('oa1', 'sqlite://memory')
+
+    assert_kind_of DataSource, @obj.oa1
+    assert @obj.oa1 = 'test'
+    assert_equal @obj.oa1, 'test'
+  end
+
+  def test_union
+    rs_first_str = "oa.sql('select id,name,age,flag from users')"
+    rs_second_str = "hr.users(:id,:name,:age,:flag)"
+
+    new_rs = @obj.union(rs_first_str, rs_second_str)
+    assert_equal 10, new_rs.size
+  end
+
+  def test_join
+    rs_first_str = "oa.sql('select id,name,age,flag from users where id in (1, 2)')"
+    rs_second_str = "hr.sql('select id, birthday from users where id in (2, 3)')"
+    condition = ":id => :id"
+    new_rs = @obj.join(rs_first_str, rs_second_str, condition)
+
+    assert_equal 2, new_rs.size
+
+    # TODO when fetch none record , then fill with nil
+  end
+
+  def test_join_2
+  end
+
+  def test_migrate_full_table_migrate
+    rs_first_str = "oa.sql('select * from users')"
+    rs_second_str = "hr.users"
+
+    # this will save the migrated resultset
+    @obj.migrate(rs_first_str, rs_second_str)
+
+    # search the db to confirm
+    rs = @obj.oa.dbh.select("select flag from users")
+
+    assert_equal 5, rs.size
+    assert_equal 'oa', rs.rows.first[0]
+  end
+
+  def test_migrate_parted_record
+    rs_first_str = "oa.sql('select id,name,flag from users where id in (2)')"
+    rs_second_str = "hr.sql('select id,name,flag from users where id in (2)')"
+
+    @obj.migrate(rs_first_str, rs_second_str)
+
+    rs = @obj.hr.dbh.select("select count(*) from users where flag='oa'")
+    assert_equal 1, rs.rows.first[0]
+  end
+
+  def test_migrate_parted_record_2
+    # include delete, updated, insert (when target record exists in db but not in rs)
+  end
+
+  private
+
+  def get_seed_sql(db, insert_times = 5)
+    insert_sql = ""
+    insert_times.times do |i|
+      insert_sql << "insert into users(id, name, age, flag) values(#{i}, 'jyp#{i}', 26, '#{db}');"
+    end
+
+    setup_sql =<<SQL
+    create table users (
+      id integer primary key not null,
+      name varchar,
+      age integer,
+      flag varchar,
+      birthday datetime default current_date
+    );
+    #{insert_sql}
+SQL
+
+    setup_sql
+  end
+end
+
+
+
+
+
+