rigrate 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 187fa5716dd1625fe30f8a5d1fcab945cf1b8293
+  data.tar.gz: 0bdffca0c4724b9d1a36b36c3a8f46dc54fd0b52
+SHA512:
+  metadata.gz: 5a5af45d6cade3df5147184238d8ad3ae41b0afdd0c399d4789fe4b6820a345205c742ba9f8ba4b9b4e1d94573d9844f0c058d1d9647c70322a420de539a8c48
+  data.tar.gz: e281b403611bef6f7fa738ad3cf64a153b2a57a91c1e35027582c47435cfb9d6003a6e801e6edc79362a33446a485f91a779d6d4f4f8a4fd7fd42efb8805169f

data/README.md

@@ -0,0 +1,160 @@
+# Rigrate
+
+## Overview
+This gem is used to migrate data between different data source in an easy way. SQLite3、MySQL、Oracle supported in current.  
+
+## Installation
+```ruby
+gem install rigrate
+```  
+
+if you are using oracle, you need install instant oracle clent at least.
+
+## Basic Usage
+
+An easy example, migrate entries hrdb users table to accounts table in oadb.
+
+Firstly， Write the migrate script
+
+```ruby
+# file : hr_users_to_oa_accounts.rigrate
+
+# define the data sources, use URI pattern like  DB://USR:PWD@HOST/DBNAME
+ds :hr, "oracle://scott:tiger@hrdb" 
+ds :oa, "mysql://root:testpwd@127.0.0.1:oadb"
+
+# migrate snippet code
+from hr.users to oa.accounts
+```
+
+Then, you can run as following:
+
+```shell
+$ rigrate execute -f hr_users_to_oa_accounts.rigrate 
+```
+
+## More Examples
+
+1. Using native SQL. Just migrate DepNO is D00001 users to target data source.
+
+    ```ruby
+    from 
+      hr.sql("select * from users where deptno = 'D00001'")
+    to
+      oa.accounts
+    ```
+
+2. Specify the sync condition columns. When the migration has condition columns, it will update target data instead of delete the records directly.
+
+    ```ruby
+    from 
+      hr.sql("select * from users")
+    to
+      oa.accounts
+    on :user_code => :job_code
+    ```
+
+3. Just migrate a part of entry source columns. In this situation migration with one condition is advised.
+
+    ```ruby
+    from 
+      hr.users(:user_code, :name, :age, :passwd)
+    to
+      oa.account(:job_code, :name, :age, :password)
+    on :user_code => :job_code
+    ```
+4. Union two data source.
+
+    ```ruby
+    from 
+      hr.users union oa.accounts 
+    to 
+      erp.users
+    ```
+5. Minus two data source
+
+    ```ruby
+    from
+      hr.users minus oa.accounts
+    to
+      erp.users
+    ```
+
+6. Join two data source. JOIN must with condition.
+
+    ```ruby
+    from
+      hr.users join oa.account on :user_code => :job_code
+    to
+      erp.users
+    ```
+    
+7. Data source select with after hooks
+
+    ```ruby
+    from
+      hr.users(:user_code, :user_name, :password, :age) do |row|
+        row[2] = "useless_password"  # make password useless
+        row[3] = 16                  # make everyone 16 forever :-)
+      end
+    to
+      oa.accounts(:job_code, :name, :pwd, :age)
+    on :user_code => :job_code
+    ```
+
+## Advance Tips
+
+1. Sync Mode  
+Rigrate support support three mode, inspired by `SyncToy` a file sync tool by Microsoft. 
+
+    **:echo** is the default mode, will delete all records of target ds which not exists in source ds  
+    **:contrbiute** is same as :echo, but keep the records even it deleted in source ds  
+    **:sync** mode will make two side of migration the same  
+    
+    use :contribute mode in migration:
+    ```ruby
+    from hr.users to oa.users mode :contribute
+    ```
+
+2. Using Transaction
+Rigrate will do not use transaction as default. Switch it on, you can do it througt command line `--strict`.
+
+    ```shell
+      $ rigrate -f script.rigrate --strict
+    ```
+
+3. Using Rigrate in other ruby script
+
+    ```ruby
+    require 'rigrate'
+    
+    str =<<EOS
+    ds :hr, "oracle://scott:tiger@hrdb" 
+    ds :oa, "mysql://root:testpwd@127.0.0.1:oadb"
+    
+    from hr.users to oa.accounts
+    EOS
+    
+    parser = Rigrate::Parser.new
+    parser.lex(str).parsing
+    ```
+4. QuickScript. a fast way to operate data, Rigrate will load a predefined data sources file in {home}/.rigrate/ds. 
+you can define some frequently used data source, and run in migration throught command line. like blow:
+
+    ```shell
+    $ rigrate execute -c "from hr.users to oa.accounts"
+    ```
+
+for more usage to check `Rigrate help`
+
+## TODO Lists
+
+1. Integrate with schedule framework
+2. Add pg/sql sever/access/csv support
+3. Row data type and length checking
+4. Rigrate file checking
+5. More logger details
+
+## Others
+
+None :-)

data/Rakefile

@@ -0,0 +1,9 @@
+gem "minitest"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc 'Run tests'
+task :default => :test

data/bin/rigrate

@@ -0,0 +1,69 @@
+#! ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && ! $LOAD_PATH.include?(lib)
+
+require 'thor'
+require 'rigrate'
+
+class RigrateApp < Thor
+  desc 'execute', 'execute a task.'
+  method_option :file, :aliases  => '-f',
+                       :type     => :string,
+                       :desc     => 'script file path.'
+  method_option :mode, :aliases => '-m',
+  					           :type	  => :string,
+                       :default => :echo,
+  					           :desc	  => "migration mode, :echo and :contribute support in current"
+  method_option :loglevel, :aliases => '-l',
+                           :type    => :string,
+                           :default => "999",
+                           :desc 	  => "Do you want to log the actions? using number 1~5 or info/warn/debug... to this option"
+  method_option :stdout, :aliases 	=> '-o',
+                          :type 	  => :boolean,
+                          :default  => false,
+                          :desc 	  => "switch on to using STDOUT in prompt."
+  method_option :strict, :aliases 	=> '-s',
+            						 :type 		=> :boolean,
+                         :default => false,
+            						 :desc 		=> "switch the option on will using db transaction to commit"
+  method_option	:script, :aliases 	=> '-c',
+            						 :type 		=> :string,
+            						 :desc 		=> "eval migration script in command line. using predefined DataSource in default ds file"
+  method_option  :dir, :aliases   => '-d',
+                       :type    => :string,
+                       :desc    => "directory of rigrate script"
+  def execute
+    # format all key from symbol to string
+    opts = options.inject({}) do |hash, (k, v)|
+      hash[k.to_sym] = v 
+      hash
+    end
+
+    opts[:loglevel] = format_loglevel(opts[:loglevel])
+    Rigrate.run(opts)
+  end
+
+  desc "verify", "verify a file is right or not"
+  method_option :file, :aliases => '-f',
+                        :desc => "the file to be verified"
+  def verify
+    
+  end
+
+  no_commands { 
+    def format_loglevel(level)
+      level = level.to_s.downcase
+      return level.to_i if level =~ /[0-5]+/
+
+      ['debug', 'info', 'warn', 'error', 'fatal', 'unknown'].each_with_index do |ele, idx|
+        return idx if ele == level.to_s.downcase
+      end
+
+      999
+    end
+  }
+  
+end
+
+RigrateApp.start

data/lib/rigrate.rb

@@ -0,0 +1,97 @@
+# encoding : utf-8
+
+require 'logger'
+
+require 'rigrate/data_source'
+require 'rigrate/error'
+require 'rigrate/interface'
+require 'rigrate/migration'
+require 'rigrate/parser'
+
+module Rigrate
+
+  ##
+  # used to output logger to mutli place
+  #
+  class MutiOutput
+    def initialize(file, prompt)
+      @target = []
+      @target << file
+      @target << STDOUT if prompt
+    end
+
+    def write(*args)
+      @target.each { |dest| dest.write(*args) }
+    end
+
+    def close
+      @target.each(&:close)
+    end
+  end
+
+  # create rigrate main path
+  path = File.join(Dir.home, ".rigrate")
+  Dir.mkdir path unless (Dir.exist? path)
+
+  @config = {
+    :loglevel => 999, # default logger not show
+    :logpath => path,
+    :mode => :echo,
+    :strict => false,
+    :stdout => false,
+    :ds => File.join(Dir.home, ".rigrate/ds"),
+    :script => nil,
+    :file => nil,
+  }
+
+  def self.run(opts = {})
+    scripts = []
+    configure(opts)
+
+    # read all scripts <- from command line , file script , rigrate dir
+    scripts << opts[:script] if opts[:script]
+    scripts << File.read(opts[:file]) if opts[:file]
+    if Dir.exist? opts[:dir].to_s     
+      Dir["#{File.join(opts[:dir], '*')}"].each do |item|
+        scripts << File.read(item)
+      end
+    end
+
+    raise RigrateError.new("your should specify one script at least.") if scripts.size <= 0
+
+    parser = Parser.new
+    # loading data source
+    if File.exist? config[:ds]
+      parser.lex(File.read(config[:ds])).parsing
+    end
+
+    scripts.each do |script|
+      parser.lex(script).parsing
+    end
+  end
+
+  def self.logger
+    logger_file = File.open(File.join(config[:logpath],logger_name), 'a')
+
+    @log = Logger.new MutiOutput.new(logger_file, config[:stdout])
+    @log.level = config[:loglevel]
+
+    @log
+  end
+
+  def self.configure(opts)
+    opts.each do |k, v|
+      raise ParserError.new("arguments #{k} not valid.") unless config.keys.include? k.to_sym
+    end
+
+    config.merge! opts
+  end
+
+  def self.config
+    @config
+  end
+
+  def self.logger_name
+    "#{Time.now.strftime('%Y-%m-%d')}.log"
+  end
+end

data/lib/rigrate/data_source.rb

@@ -0,0 +1,43 @@
+# encoding : utf-8
+
+require 'uri'
+
+module Rigrate
+  class DataSource
+    attr_accessor :dbh
+
+    def initialize(conn_uri)
+      uri = URI.parse(conn_uri)
+      @dbh = eval(uri.scheme.capitalize).new conn_uri
+    end
+
+    def sql(str, *args)
+      begin
+        @dbh.select(str, *args)
+      rescue SQLite3::SQLException => e
+        puts "DB: #{@dbh.inspect} SQL: #{str}"
+        raise e
+      end
+    end
+
+    def method_missing(mth, *args, &block)
+      begin
+        table_name = mth
+        columns = args
+
+        str_sql = build_sql(table_name, *columns)
+        @dbh.select(str_sql, &block)
+      rescue Exception => e
+        raise e
+      end
+    end
+
+    private
+
+    def build_sql(table, *columns)
+      columns = ['*'] if columns.size == 0
+
+      "select #{columns.map(&:to_s).join(',')} from #{table}"
+    end
+  end
+end

data/lib/rigrate/error.rb

@@ -0,0 +1,11 @@
+# encoding : utf-8
+
+module Rigrate
+  class BasicError < ::StandardError; end
+
+  class RigrateError < BasicError; end
+  class ParserError < BasicError; end
+  class InterfaceError < BasicError; end
+  class ResultSetError < BasicError; end
+  class DirverError < BasicError; end
+end

data/lib/rigrate/interface.rb

@@ -0,0 +1,20 @@
+# encoding : utf-8
+
+require 'uri'
+require 'rigrate/interface/driver'
+require 'rigrate/interface/result_set'
+require 'rigrate/interface/row'
+require 'rigrate/interface/sqlite'
+require 'rigrate/interface/mysql'
+require 'rigrate/interface/oracle'
+
+module Rigrate
+  def self.lazy_load_driver(driver_name)
+    driver_path = "./interface/#{driver_name}"
+    if File.exists? driver_path
+      raise InterfaceError.new('Driver not found.')
+    end
+
+    require driver_path
+  end
+end

data/lib/rigrate/interface/driver.rb

@@ -0,0 +1,41 @@
+# encoding : utf-8
+
+module Rigrate
+  # defination for column include name and type
+  # column name is a string
+  Column = Struct.new(:name, :type)
+
+  class Driver
+    attr_accessor :db
+
+    def connect(db)
+      self.db = db
+    end
+
+    def method_missing(mth, *args, &block)
+      @db.send(mth, *args, &block)
+    end
+
+    def extract_conn_param(uri)
+      opts = {}
+      opts['db_type'] = uri.scheme if uri.scheme
+      opts['host'] = uri.host if uri.host
+      opts['username'] = uri.user if uri.user
+      opts['password'] = uri.password if uri.password
+      opts['port'] = uri.port if uri.port
+      opts['db_name'] = uri.path.tr('/','') if uri.path.tr('/','').size > 0
+
+      opts
+    end
+
+    def to_native_row(row, column_info)
+      row
+    end
+
+    def extract_tbl_from_sql(sql_str)
+      return $1 if sql_str =~ /from\s+(\w*)\s*/
+
+      raise Exception.new('a lastest one table name must specify.')
+    end
+  end
+end