data_transit 0.2.0

data/Database.yml

@@ -0,0 +1,27 @@
+source:
+  adapter: oracle_enhanced
+  host: 10.162.211.9
+  port: 1521
+  database: ztjc2
+  username: mw_ztjc
+  password: ztjc
+
+target:
+  adapter: oracle_enhanced
+  host: 10.162.106.153
+  port: 1521
+  database: orcl
+  username: mw_app
+  password: mw_app
+  
+test_target:
+  adapter: sqlite3
+  database: development.sqlite3
+  pool: 5
+  timeout: 5000
+
+production: 
+  adapter: oracle 
+  database: comics_catalog_production  
+  username: comics_catalog 
+  password:

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 tianyuan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README

@@ -0,0 +1,150 @@
+data_transit
+data_transit is a ruby gem/app used to migrate between different relational 
+databases, supporting customized migration procedure.
+
+
+#1 Introduction
+data_transit relies on activerecord to generate database Models on the fly. 
+Tt is executed within a database transaction, and should any error occur during
+data transit, it will cause the transaction to rollback. So don't worry about
+introducing dirty data into your target database.
+
+
+#2 Install
+data_transit can be installed using gem
+    gem install data_transit
+or 
+    download data_transit.gem
+    gem install --local /where/you/put/data_transit.gem
+
+In your command line, input data_transit, You can proceed to "how to use" if you
+can see message prompt as below.
+data_transit
+usage: data_transit command args. 4 commands are listed below...
+
+
+#3 How to use
+
+#3.1 Config DataBase Connection
+    data_transit setup_db_conn /your/yaml/database/config/file
+
+Your db config file should be compatible with the activerecord adapter you are 
+using and configured properly. As the name suggests, source designates which 
+database you plan to copy data from, while target means the copy destination.
+Note the key 'source' and 'target' should be kept unchanged!
+For example, here is sample file for oracle dbms.
+
+database.yml
+source:#don't change this line
+  adapter: oracle_enhanced
+  database: dbserver
+  username: xxx
+  password: secret
+
+target:#don't change this line
+  adapter: oracle_enhanced
+  database: orcl
+  username: copy1
+  password: cipher
+
+
+#3.2 Create Database Schema(optional)
+If you can have your target database schema created, move to 3.3 "copy data", 
+otherwise use your database specific tools to generate an identical schema in 
+your target database. Or if you don't have a handy tool to generate the schema, 
+for instance when you need to migrate between different database systems, you 
+can use data_transit to dump a schema description file based on your source 
+database schema, and then use this file to create your target schema.
+
+Note this gem is built on activerecord, so it should work well for the database 
+schema compatible with rails conventions. Example, a single-column primary key  
+rather than a compound primary key (primary key with more than one columns), 
+primary key of integer type instead of other types like guid, timestamp etc.
+
+In data_transit, I coded against the situation where non-integer is used, 
+therefore resulting a minor problem that the batch-query feature provided by 
+activerecord can not be used because of its dependency on integer primary key.
+In this special case, careless selection of copy range might overburden network,
+database server because all data in the specified range are transmitted from the
+source database and then inserted into the target database.
+
+
+#3.2.1 Dump Source Database Schema
+    data_transit dump_schema [schema_file] [rule_file]
+[schema_file] will be used to contain dumped schema, [rule_file] describes how 
+your want data_transit to copy your data.
+
+Note if your source schema is somewhat a legacy, you might need some manual work
+to adjust the generated schema_file to better meet your needs. 
+
+For example, in my test, the source schema uses obj_id as id, and uses guid as 
+primary key type, so I need to impede activerecord auto-generating "id" column 
+by removing primary_key => "obj_id" and adding :id=>false, and then appending 
+primary key constraint in the end of each table definition. See below.
+
+here is an example dumped schema file
+
+ActiveRecord::Schema.define(:version => 0) do
+  create_table "table_one", primary_key => "obj_id", :force => true do |t|
+    #other fields
+  end
+  #other tables
+end
+
+and I manually changed the schema definition to
+
+ActiveRecord::Schema.define(:version => 0) do
+  create_table "table_one", :id => false, :force => true do |t|
+    t.string "obj_id",           :limit => 42  
+    #other fields
+  end
+  execute "alter table table_one add primary key(obj_id)"
+end
+
+#3.2.2 Create Target Database Schema
+    data_transit create_table [schema_file]
+If everything goes well, you will see a bunch of ddl execution history.
+
+
+#3.3 Copy Data
+    data_transit copy_data [rule_file]
+[rule_file] contains your copy logic. For security reasons, I changed table names
+and it looks as follows.
+
+
+start_date = "2015-01-01 00:00:00"
+end_date = "2015-02-01 00:00:00"
+
+migrate do
+  choose_table "APP.TABLE1","APP.TABLE2","APP.TABLE3","APP.TABLE4","APP.TABLE5","APP.TABLE6"
+  batch_by "ACQUISITION_TIME BETWEEN TO_DATE('#{start_date}','yyyy-mm-dd hh24:mi:ss') AND TO_DATE('#{end_date}', 'yyyy-mm-dd hh24:mi:ss')"
+  register_primary_key 'OBJ_ID'
+end
+
+migrate do
+  choose_table "APP.TABLE7","APP.TABLE8","APP.TABLE9","APP.TABLE10","APP.TABLE11","APP.TABLE12"
+  batch_by "ACKTIME BETWEEN TO_DATE('#{start_date}','yyyy-mm-dd hh24:mi:ss') AND TO_DATE('#{end_date}', 'yyyy-mm-dd hh24:mi:ss')"
+  register_primary_key 'OBJ_ID'
+end
+
+migrate do
+  choose_table "APP.TABLE13","APP.TABLE14","APP.TABLE15","APP.TABLE16","APP.TABLE17","APP.TABLE18"
+  batch_by "1>0" #query all data because these tables don't have a reasonable range
+  register_primary_key 'OBJ_ID'
+  pre_work do |targetCls| targetCls.delete_all("1>0")  end #delete all in target
+  #post_work do |targetCls|  end
+end
+
+
+Each migrate block contains a data_transit task. 
+
+"choose_table" describes which tables are included in this task. These tables 
+share some nature in common, and can be processed with the same rule. 
+
+"batch_by" is the query condition
+
+"register_primary_key" describes the primary key of the tables.
+
+"pre_work" is a block executed before each table is processed.
+
+"post_work" is a block executed after each table is processed.

data/Rakefile

@@ -0,0 +1,47 @@
+# 
+# To change this license header, choose License Headers in Project Properties.
+# To change this template file, choose Tools | Templates
+# and open the template in the editor.
+ 
+
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'rubygems/package_task'
+require 'rdoc/task'
+require 'rake/testtask'
+
+spec = Gem::Specification.new do |s|
+  s.name = 'data_transit'
+  s.version = '0.2.0'
+  s.has_rdoc = true
+  s.extra_rdoc_files = ['README', 'LICENSE']
+  s.summary = 'a ruby gem/app used to migrate between databases, supporting customized migration procedure'
+  s.description = 'data_transit relies on activerecord to generate database Models on the fly. Tt is executed within a database transaction, and should any error occur during data transit, it will cause the transaction to rollback. So don\'t worry about introducing dirty data into your target database'
+  s.author = 'thundercumt'
+  s.email = 'thundercumt@126.com'
+  s.homepage = 'https://github.com/thundercumt/data_transit'
+  s.executables << "data_transit"
+  s.files = %w(LICENSE README Rakefile Database.yml) + Dir.glob("{bin,lib,spec}/**/*")
+  s.require_path = "lib"
+  s.bindir = "bin"
+end
+
+Gem::PackageTask.new(spec) do |p|
+  p.gem_spec = spec
+  p.need_tar = true
+  p.need_zip = true
+end
+
+Rake::RDocTask.new do |rdoc|
+  files =['README', 'LICENSE', 'lib/**/*.rb']
+  rdoc.rdoc_files.add(files)
+  rdoc.main = "README" # page to start on
+  rdoc.title = "Data_Transit Docs"
+  rdoc.rdoc_dir = 'doc/rdoc' # rdoc output folder
+  rdoc.options << '--line-numbers'
+end
+
+Rake::TestTask.new do |t|
+  t.test_files = FileList['test/**/*.rb']
+end

data/bin/data_transit

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+require File::expand_path('../../lib/datatransit/cli', __FILE__)
+
+command = ARGV[0]
+
+help_msg = "usage: data_transit command args. 4 commands are listed below\n
+1 data_transit setup_db_conn [conn.yml]
+conn.yml: the yml file describing db connection and adapters\n
+2 data_transit dump_schema [schema_file] [rule_file]
+schema_file: the file to store source database schema
+rule_file: the user migration rule\n
+3 data_transit create_table [schema_file]
+schema_file: the file generated by dump_schema\n
+4 data_transit copy_data [rule_file]
+rule_file: the user migration rule"
+ 
+case command
+when "setup_db_conn"
+  yml = ARGV[1]
+  if File::exists?(yml)
+    setup_db_conn yml
+  else
+    print "file #{yml} does not exist!\n"
+  end
+  
+when "dump_schema"
+  schema = ARGV[1]
+  rule = ARGV[2]
+  if not (schema =~ /\.rb$/)
+    schema += ".rb"
+  end
+  dump_schema schema, rule
+when "create_table"
+  create_tables ARGV[1]
+when "copy_data"
+  copy_data ARGV[1]
+else
+  puts help_msg
+end

data/bin/data_transit.bat

@@ -0,0 +1,6 @@
+@ECHO OFF
+IF NOT "%~f0" == "~f0" GOTO :WinNT
+@"ruby.exe" "data_transit" %1 %2 %3 %4 %5 %6 %7 %8 %9
+GOTO :EOF
+:WinNT
+@"ruby.exe" "data_transit" %*

data/lib/datatransit/clean_find_in_batches.rb

@@ -0,0 +1,34 @@
+module CleanFindInBatches
+
+  def self.included(base)
+    base.class_eval do
+      alias :old_find_in_batches :find_in_batches
+      alias :find_in_batches :replacement_find_in_batches
+    end
+  end
+
+  # Override due to implementation of regular find_in_batches
+  # conflicting using UUIDs
+  def replacement_find_in_batches(options = {}, &block)
+    relation = self
+    return old_find_in_batches(options, &block) if relation.primary_key.is_a?(Arel::Attributes::Integer)
+    # Throw errors like the real thing
+    if (finder_options = options.except(:batch_size)).present?
+      raise "You can't specify an order, it's forced to be #{batch_order}" if options[:order].present?
+      raise "You can't specify a limit, it's forced to be the batch_size" if options[:limit].present?
+      raise 'You can\'t specify start, it\'s forced to be 0 because the ID is a string' if options.delete(:start)
+      relation = apply_finder_options(finder_options)
+    end
+    # Compute the batch size
+    batch_size = options.delete(:batch_size) || 1000
+    offset = 0
+    # Get the relation and keep going over it until there's nothing left
+    relation = relation.except(:order).order(batch_order).limit(batch_size)
+    while (results = relation.offset(offset).limit(batch_size).all).any?
+      block.call results
+      offset += batch_size
+    end
+    nil
+  end
+
+end

data/lib/datatransit/cli.rb

@@ -0,0 +1,66 @@
+# To change this license header, choose License Headers in Project Properties.
+# To change this template file, choose Tools | Templates
+# and open the template in the editor.
+
+#require 'rubygems'
+#require 'bundler/setup'
+#Bundler.require
+
+#initial code
+ENV['NLS_LANG'] = 'SIMPLIFIED CHINESE_CHINA.ZHS16GBK' if ENV['NLS_LANG'] == nil
+# It is recommended to set time zone in TZ environment variable so that the same timezone will be used by Ruby and by Oracle session
+ENV['TZ'] = 'UTC'
+
+require 'active_record'
+require File::expand_path('../clean_find_in_batches', __FILE__)
+ActiveRecord::Batches.send(:include, CleanFindInBatches)
+
+require File::expand_path('../database', __FILE__)
+require File::expand_path('../model/tables_source', __FILE__)
+require File::expand_path('../model/tables_target', __FILE__)
+require File::expand_path('../model_dumper', __FILE__)
+require File::expand_path('../rule_dsl', __FILE__)
+  
+def dump_schema schema_file, rule_file
+  puts "preparing to generate schema.rb, the schema of source database\n"
+  ActiveRecord::Base.establish_connection(DataTransit::Database.source)
+  #tables = DataTransit::Database.tables
+  worker = DTWorker.new rule_file
+  worker.load_work
+  tables = worker.get_all_tables
+
+  print tables, "\n"
+
+  dumper = DataTransit::ModelDumper.new tables
+  dumper.dump_tables File.open(schema_file, 'w');
+  ActiveRecord::Base.remove_connection
+  puts "schema.rb generated\n"
+end
+  
+def create_tables schema_file
+  puts "\npreparing to create tables in the target database\n"
+  ActiveRecord::Base.establish_connection(DataTransit::Database.target)
+  require schema_file
+  ActiveRecord::Base.remove_connection
+  puts "tables created in target database\n"
+end
+  
+  
+def copy_data rule_file
+  worker = DTWorker.new rule_file
+  worker.load_work
+  worker.do_work
+end
+
+def setup_db_conn db_yml
+  if File::exists? db_yml
+    src = File::open(db_yml, "r")
+    dst = File::open( File.expand_path('../../../database.yml', __FILE__), "w" )
+    
+    src.each_line { |line|
+       dst.write line
+    }
+    src.close
+    dst.close
+  end
+end

data/lib/datatransit/database.rb

@@ -0,0 +1,21 @@
+# To change this license header, choose License Headers in Project Properties.
+# To change this template file, choose Tools | Templates
+# and open the template in the editor.
+
+module DataTransit
+
+class Database
+    
+    @@dbconfig = YAML::load( File.open( File.expand_path('../../../database.yml', __FILE__) ) )
+    
+    def self.source
+      @@dbconfig['source']
+    end
+    
+    def self.target
+      @@dbconfig['target']
+    end
+  
+  end
+  
+end

data/lib/datatransit/helper.rb

@@ -0,0 +1,26 @@
+# To change this license header, choose License Headers in Project Properties.
+# To change this template file, choose Tools | Templates
+# and open the template in the editor.
+
+module TimeExt
+  def fly_by_day days
+    Time.at(self.to_i + days * 24 * 60 * 60)
+  end
+  
+  def fly_by_week weeks
+    Time.at(self.to_i + weeks * 7 * 24 * 60 * 60)
+  end
+  
+  def fly_by_month months
+    Time.at(self.to_i + months * 30 * 24 * 60 * 60)
+  end
+  
+  def midnight
+    Time.at(self.year, self.month, self.day, 24, 0, 0 )
+  end
+  
+end
+
+class Time
+  include TimeExt
+end

data/lib/datatransit/model/tables_source.rb

@@ -0,0 +1,18 @@
+# To change this license header, choose License Headers in Project Properties.
+# To change this template file, choose Tools | Templates
+# and open the template in the editor.
+
+require 'active_record'
+require File::expand_path('../../database', __FILE__)
+
+module DataTransit
+
+module Source
+  
+    class SourceBase < ActiveRecord::Base
+      self.abstract_class = true
+      establish_connection DataTransit::Database.source
+    end
+  end
+
+end

data/lib/datatransit/model/tables_target.rb

@@ -0,0 +1,18 @@
+# To change this license header, choose License Headers in Project Properties.
+# To change this template file, choose Tools | Templates
+# and open the template in the editor.
+
+require 'active_record'
+require File::expand_path('../../database', __FILE__)
+
+module DataTransit
+  
+  module Target
+    
+    class TargetBase < ActiveRecord::Base
+      self.abstract_class = true
+      establish_connection DataTransit::Database.target
+    end
+  end
+
+end

data/lib/datatransit/model_dumper.rb

@@ -0,0 +1,65 @@
+# To change this license header, choose License Headers in Project Properties.
+# To change this template file, choose Tools | Templates
+# and open the template in the editor.
+
+module DataTransit
+  class GivenTableDumper < ActiveRecord::SchemaDumper
+    
+    def self.give_tables tables
+      @@tables = tables
+    end
+    
+    def dump(stream)
+=begin      
+      if all_tables_exist?
+        header(stream)
+        tables(stream)
+        trailer(stream)
+      else
+        print "No schema generated, because some table[s] do not exist!\n"
+      end
+=end
+      header(stream)
+      tables(stream)
+      trailer(stream)
+      
+      stream
+    end
+    
+    def tables(stream)
+      @@tables.each do |tbl|
+        table(tbl, stream)
+      end
+    end
+    
+  private
+    def all_tables_exist?
+      all_tables = @connection.tables
+      print all_tables, '!!!!!!!!!!!!!!!!!!!!111'
+      tables_all_exist = true
+      
+      @@tables.each do |tbl|
+        unless all_tables.include? tbl
+          print "table [", tbl, "] doesn't exist!\n" 
+          tables_all_exist = false
+        end
+      end
+      
+      tables_all_exist
+    end
+  end
+  
+  
+  class ModelDumper
+    
+    def initialize table_list
+      @tables = table_list
+    end
+    
+    def dump_tables (stream)
+      GivenTableDumper.give_tables @tables
+      GivenTableDumper.dump ActiveRecord::Base.connection, stream
+    end
+  end
+  
+end

data/lib/datatransit/rule_dsl.rb

@@ -0,0 +1,235 @@
+# To change this license header, choose License Headers in Project Properties.
+# To change this template file, choose Tools | Templates
+# and open the template in the editor.
+
+#chain_filter1 --> chain_filter2 --> chain_filter3
+
+require 'singleton'
+require 'progress_bar'
+
+#require File::expand_path('../database', __FILE__)
+require File::expand_path('../model/tables_source', __FILE__)
+require File::expand_path('../model/tables_target', __FILE__)
+
+module RULE_DSL
+  
+  class Task
+    attr_accessor :tables, :search_cond, :pk, :pre_work, :post_work, :filter
+  end
+  
+  class TaskSet
+    def initialize
+      @tasks = []
+    end
+    
+    def push_back task
+      @tasks << task
+    end
+    
+    def pop_head
+      @tasks.delete_at 0
+    end
+    
+    def first_task
+      @tasks[0]
+    end
+    
+    def last_task
+      @tasks[@tasks.length - 1]
+    end
+    
+    def length
+      @tasks.length
+    end
+    
+    def task_at idx
+      @tasks[idx]
+    end
+    
+    def each 
+      for i in 0 .. @tasks.length-1
+        yield @tasks[i]
+      end
+    end
+  end
+  
+  def load_rule *rule_file
+    rule_file.each{ |file| self.instance_eval File.read(file), file }
+  end
+  
+  def migrate &script
+    @taskset ||= TaskSet.new
+    task = Task.new
+    @taskset.push_back task
+    yield 
+  end
+  
+  def choose_table *tables
+    task = @taskset.last_task
+    task.tables = tables
+  end
+
+  def batch_by search_cond
+    task = @taskset.last_task
+    task.search_cond = search_cond
+  end
+  
+  def register_primary_key *key
+    task = @taskset.last_task
+    task.pk = key.map(&:downcase)
+  end
+
+  def filter_out_with &filter
+    task = @taskset.last_task
+    task.filter = filter
+  end
+  
+  def pre_work &pre
+    task = @taskset.last_task
+    task.pre_work = pre
+  end
+  
+  def post_work &post
+    task = @taskset.last_task
+    task.post_work = post
+  end
+  
+  def get_all_tables
+    tables = []
+    @taskset.each { |task| tables << task.tables }
+    tables.flatten!
+  end
+end
+
+class DTWorker
+  include RULE_DSL
+  attr_accessor :batch_size
+  
+  def initialize rule_file
+    @rule_file = rule_file
+    @batch_size = 500
+  end
+  
+  def load_work
+    load_rule @rule_file
+  end
+  
+  def do_work
+    #we copy all or nothing. a mess is not welcome here
+    #on failures, transaction will rollback automatically
+    DataTransit::Target::TargetBase.transaction do
+      @taskset.each do |task|
+        do_task task
+      end
+    end
+  end
+  
+  def do_task task
+    pks = task.pk#a context-free-variable in context switches
+    tables = task.tables
+    
+    tables.each do |tbl|
+      sourceCls = Class.new(DataTransit::Source::SourceBase) do
+        self.table_name = tbl
+      end
+
+      #columns = sourceCls.columns.map(&:name).map(&:downcase)
+      columns = sourceCls.columns
+      pk_column = get_pk_column(columns, pks)
+      if pk_column != nil
+        pk, pk_type = pk_column.name, pk_column.type
+      else
+        pk, pk_type = nil, nil
+      end
+
+      sourceCls.instance_eval( "self.primary_key = \"#{pk}\"") if pk != nil
+
+      targetCls = Class.new(DataTransit::Target::TargetBase) do
+        self.table_name = tbl
+      end
+      targetCls.instance_eval( "self.primary_key = \"#{pk}\"") if pk != nil
+
+      print "\ntable ", tbl, ":\n"
+      do_user_ar_proc targetCls, task.pre_work if task.pre_work
+      do_batch_copy sourceCls, targetCls, task, pk, pk_type=="integer"
+      do_user_ar_proc targetCls, task.post_work if task.post_work
+    end
+  end
+  
+  def do_user_ar_proc targetCls, proc
+    proc.call targetCls
+  end
+  
+  def do_batch_copy (sourceCls, targetCls, task, pk = nil, in_batch = false)
+    count = sourceCls.where(task.search_cond).size.to_f
+    return if count <= 0
+    
+    how_many_batch = (count / @batch_size).ceil
+    #the progress bar
+    bar = ProgressBar.new(count)
+    
+    if in_batch 
+      0.upto (how_many_batch-1) do |i|  
+        sourceCls.where(task.search_cond).find_each(
+          start: i * @batch_size, batch_size: @batch_size) do |source_row|
+          
+          #update progress
+          bar.increment!
+          
+          if task.filter
+            next if do_filter_out task.filter, source_row
+          end
+          target_row = targetCls.new source_row.attributes
+
+          #activerecord would ignore pk field, and the above initialization will result nill primary key.
+          #here the original pk is used in the target_row, it is what we need exactly.
+          if pk 
+            target_row.send( "#{pk}=", source_row.send("#{pk}") )
+          end
+
+          target_row.save
+        end
+      end
+    else
+      sourceCls.where(task.search_cond).each do |source_row|
+        #update progress
+        bar.increment!
+        
+        if task.filter
+          next if do_filter_out task.filter, source_row
+        end
+        target_row = targetCls.new source_row.attributes
+
+        #activerecord would ignore pk field, and the above initialization will result nill primary key.
+        #here the original pk is used in the target_row, it is what we need exactly.
+        if pk 
+          target_row.send( "#{pk}=", source_row.send("#{pk}") )
+        end
+
+        target_row.save
+      end
+    end
+    
+    
+  end
+  
+  def do_filter_out filter, row
+    if filter.call row
+      return true
+    end
+    false
+  end
+  
+  private
+  def get_pk_column(columns, given_pk)
+    column_names = columns.map(&:name).map(&:downcase)
+    pk = column_names & given_pk
+    if pk && pk.length > 0
+      pk = pk[0]
+      return columns[column_names.index(pk)]
+    else
+      return nil
+    end
+  end
+  
+end

data/lib/datatransit/tasks.rb

@@ -0,0 +1,71 @@
+# To change this license header, choose License Headers in Project Properties.
+# To change this template file, choose Tools | Templates
+# and open the template in the editor.
+require 'rubygems'
+require 'bundler/setup'
+Bundler.require
+
+#initial code
+ENV['NLS_LANG'] = 'SIMPLIFIED CHINESE_CHINA.ZHS16GBK' if ENV['NLS_LANG'] == nil
+# It is recommended to set time zone in TZ environment variable so that the same timezone will be used by Ruby and by Oracle session
+ENV['TZ'] = 'UTC'
+
+
+namespace :db  do
+  
+  desc "environment verification"
+  task :environment do
+    require 'active_record'
+    #require 'activerecord-oracle_enhanced-adapter'
+    #require 'oci8'
+
+    require File::expand_path('../database', __FILE__)
+    require File::expand_path('../model/tables_source', __FILE__)
+    require File::expand_path('../model/tables_target', __FILE__)
+    require File::expand_path('../model_dumper', __FILE__)
+  end
+  
+  desc "dsl related definitions"
+  task :dsl => :environment do
+    require File::expand_path('../rule_dsl', __FILE__)
+  end
+  
+  desc "generate schema.rb, the schema of source database"
+  task :dump_schema => :dsl do
+    puts "preparing to generate schema.rb, the schema of source database\n"
+    ActiveRecord::Base.establish_connection(DataTransit::Database.source)
+    #tables = DataTransit::Database.tables
+    worker = DTWorker.new File::expand_path('../../rule.rb', __FILE__)
+    worker.load_work
+    tables = worker.tables
+    
+    print tables, "\n"
+    
+    dumper = DataTransit::ModelDumper.new tables
+    dumper.dump_tables File.open(File::expand_path('../schema.rb', __FILE__), 'w');
+    ActiveRecord::Base.remove_connection
+    puts "schema.rb generated\n"
+  end
+  
+  desc "use schema.rb to schema created in target database"
+  task :create_tables => :environment do
+    puts "\npreparing to create tables in the target database\n"
+    ActiveRecord::Base.establish_connection(DataTransit::Database.target)
+    require File.expand_path("../schema", __FILE__)
+    ActiveRecord::Base.remove_connection
+    puts "tables created in target database\n"
+  end
+  
+  desc "data transit, copy rows from source db tables to target db tables\n it supports incremental copy by additional arguments"
+  task :copy_data, [] => :dsl do
+    worker = DTWorker.new File::expand_path('../../rule.rb', __FILE__)
+    worker.do_work
+  end
+  
+  #task :my, [:arg1, :arg2, :arg3, :arg4] => :environment do |t, args|
+    #print t, args
+    #tables = DataTransit::Database.tables
+    #print tables
+  #end
+  
+end

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification
+name: data_transit
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+  prerelease: 
+platform: ruby
+authors:
+- thundercumt
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-12 00:00:00.000000000 Z
+dependencies: []
+description: data_transit relies on activerecord to generate database Models on the
+  fly. Tt is executed within a database transaction, and should any error occur during
+  data transit, it will cause the transaction to rollback. So don't worry about introducing
+  dirty data into your target database
+email: thundercumt@126.com
+executables:
+- data_transit
+extensions: []
+extra_rdoc_files:
+- README
+- LICENSE
+files:
+- LICENSE
+- README
+- Rakefile
+- Database.yml
+- bin/data_transit
+- bin/data_transit.bat
+- lib/datatransit/clean_find_in_batches.rb
+- lib/datatransit/cli.rb
+- lib/datatransit/database.rb
+- lib/datatransit/helper.rb
+- lib/datatransit/model/tables_source.rb
+- lib/datatransit/model/tables_target.rb
+- lib/datatransit/model_dumper.rb
+- lib/datatransit/rule_dsl.rb
+- lib/datatransit/tasks.rb
+homepage: https://github.com/thundercumt/data_transit
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.28
+signing_key: 
+specification_version: 3
+summary: a ruby gem/app used to migrate between databases, supporting customized migration
+  procedure
+test_files: []