myreplicator 1.1.0 → 1.1.1

data/lib/configuration.rb~

@@ -1,23 +0,0 @@
-module Myreplicator
-  ##
-  # Configuration class for avoiding load the yml file everytime and cleaner config settings
-  ##
-  class Configuration
-    class << self; attr_accessor :tmp_path, :mysqldump, :mysql end
-    
-    # yml = YAML.load(File.read("#{Myreplicator.app_root}/config/myreplicator.yml"))
-    # Kernel.p yml["myreplicator"]
-
-    # @@tmp_path = yml["myreplicator"]["tmp_path"]
-    # @@mysql = yml["myreplicator"]["mysql"]
-    # @@mysqldump = yml["myreplicator"]["mysqldump"]
-  end
-
-  def self.config(&block)
-    @@config ||= Myreplicator::Configuration.new
-
-    yield @@config if block
-
-    return @@config
-  end
-end

data/lib/exporter/mysql_exporter.rb~

@@ -1,6 +0,0 @@
-module Myreplicator
-  class MysqlExporter
-    
-    
-  end
-end

data/lib/exporter.rb~

@@ -1 +0,0 @@
-require ""

data/lib/loader/vertica/source_db.rb~

@@ -1,30 +0,0 @@
-module VerticaUtils
-  class SourceDb < ActiveRecord::Base
-      
-    def self.connect db
-      establish_connection(ActiveRecord::Base.configurations[db])
-    end
-  
-    ##
-    # Returns tables as an Array
-    # releases the connection
-    ##
-    def self.get_tables(db)
-      tables = []
-      begin
-        self.connect(db)
-        tables = self.connection.tables
-        self.connection_pool.release_connection
-      rescue Mysql2::Error => e
-        puts "Connection to #{db} Failed!"
-        puts e.message
-      end
-      return tables
-    end
-  
-    def self.exec_sql source_db,sql
-      SourceDb.connect(source_db)
-      return SourceDb.connection.execute(sql)
-    end
-  end
-end

data/lib/loader/vertica/vertica_loader.rb~

@@ -1,175 +0,0 @@
-require 'vertica_utils/types'
-require 'vertica_utils/source_db'
-require 'vertica_utils/vertica_sql'
-
-module VerticaUtils
-  class VerticaLoader
-    class << self
-      def create_table *args
-        options = args.extract_options!
-        columns = []
-        table_definition(options).each(:as => :hash) do |row|
-          columns << row
-        end
-        options[:columns] = columns
-      
-        sql = VerticaUtils::VerticaSql.create_table_stmt options
-      end
-    
-      def create_all_tables db
-        tables = VerticaUtils::SourceDb.get_tables(db)
-        sqls = {}
-        tables.each do |table|
-          puts "Creating #{db}.#{table}"
-          sql = "DROP TABLE IF EXISTS #{db}.#{table} CASCADE;"
-          VerticaDb::Base.connection.execute sql
-          sql = VerticaUtils::VerticaLoader.create_table(:vertica_db => "bidw",
-          :vertica_table => table,
-          :vertica_schema => db,
-          :table => table,
-          :db => db)
-          sqls["#{table}"] = sql
-          VerticaDb::Base.connection.execute sql
-        end
-      end
-    
-      def prepare_options *args
-        options = args.extract_options!
-        options.reverse_merge!(
-          :host => "sfo-load-dw-01",
-          :user => "vertica",
-          :pass => "test",
-          :db => "bidw",
-          :schema => "test",
-          :table => "",
-          :file => "",
-          :delimiter => "\t",
-          :null_value => "NULL",
-          :enclosed => ""
-        )
-        return options  
-      end
-      
-      # VerticaUtils::VerticaLoader.load_to_vertica({:schema => "king", :table => "category_overview_data", :file => "tmp/vertica/category_overview_data.tsv", :null_value => "NULL"})
-      def load_to_vertica *args
-        list_of_nulls =  ["0000-00-00"]
-        options = args.extract_options!
-        prepared_options = prepare_options options
-        Kernel.p prepared_options[:file]
-        if prepared_options[:file].blank?
-          raise "No input file"
-        end
-      
-        begin
-          process_file(:file => prepared_options[:file], :list_of_nulls => list_of_nulls, :null_value => prepared_options[:null_value])
-          cmd = get_vsql_command(prepared_options)
-          Kernel.p cmd
-          system(cmd)
-        rescue Exception => e
-          raise e.message
-        end
-      end
-        
-      def get_vsql_command prepared_options
-        file_extension = prepared_options[:file].split('.').last
-        file_handler = ""
-        file_handler = "GZIP" if file_extension == "gz" 
-        sql = "COPY #{prepared_options[:schema]}.#{prepared_options[:table]} FROM LOCAL \'#{prepared_options[:file]}\' #{file_handler} DELIMITER E\'#{prepared_options[:delimiter]}\' NULL as \'#{prepared_options[:null_value]}\' ENCLOSED BY \'#{prepared_options[:enclosed]}\' EXCEPTIONS 'tmp/vertica/load_exceptions.log';"
-        cmd = "/opt/vertica/bin/vsql -h #{prepared_options[:host]} -U #{prepared_options[:user]} -w #{prepared_options[:pass]} -d #{prepared_options[:db]} -c \"#{sql}\""
-        return cmd
-      end
-      
-      def process_file *args
-        ### replace the null values in the input file 
-        options = args.extract_options!
-        options[:file].blank? ? return : file = options[:file]
-        options[:list_of_nulls].blank? ? list_of_nulls = [] : list_of_nulls = options[:list_of_nulls]
-        options[:null_value].blank? ? null_value = "NULL" : null_value = options[:null_value]
-        
-        file_extension = file.split('.').last
-        case file_extension
-        when "tsv", "csv"
-          process_flat_file(file, list_of_nulls, null_value)
-        when "gz"
-          process_gzip_file(file, list_of_nulls, null_value)
-        else
-          raise "Un supported file extension"
-        end
-      end
-      
-      def replace_null(file, list_of_nulls, null_value = "NULL")
-        list_of_nulls.each do | value|
-          # special case for NULL MySQL datetime/date type but the column is defined NOT NULL
-          if value == '0000-00-00'
-            cmd1 = "sed -i 's/#{value}/1900-01-01/g' #{file}"
-            Kernel.p cmd1
-            system(cmd1)
-          else
-            cmd1 = "sed -i 's/#{value}/#{null_value}/g' #{file}"
-            Kernel.p cmd1
-            system(cmd1)
-          end
-        end
-      end
-      
-      def process_flat_file file, list_of_nulls, null_value 
-        # sed
-        replace_null(file, list_of_nulls, null_value)
-      end
-      
-      def process_gzip_file file, list_of_nulls, null_value
-        # unzip
-        cmd = "gunzip -f #{file} -c > tmp/temp.txt"
-        system(cmd)
-        # sed
-        replace_null(file, list_of_nulls, null_value)
-        # zip
-        cmd2 = "gzip tmp/temp.txt -c > #{file}"
-        system(cmd2)
-      end
-        
-      def get_table_columns * args
-        options = args.extract_options!
-        if options[:host].blank? or options[:user].blank?  or options[:pass].blank? or options[:db].blank? or options[:schema].blank? or options[:table].blank? 
-          raise "Unspecified host, user, pass, db, schema or table"
-        end
-        result = {}
-        count = 0
-        begin
-          cmd = "/opt/vertica/bin/vsql -h #{options[:host]} -U #{options[:user]} -w #{options[:pass]} -d #{options[:db]} -c '\\d #{options[:schema]}.#{options[:table]}';"
-          puts cmd
-          output = `#{cmd}`
-          lines = output.split("\n")
-          data = lines[3..lines.size-2]
-          data.each do |item|
-            count += 1
-            column_name = item.split("|")[2].lstrip!.rstrip!
-            column_type = item.split("|")[3].lstrip!.rstrip!
-            puts column_name + " " + column_type
-            result[count] = {column_name => column_type}
-          end
-        rescue Exception => e
-          raise e.message
-        end
-        return result
-      end
-        
-      def ssh_connection options
-        ssh = Net::SSH.start(options[:ssh_host], options[:ssh_user], :password => options[:ssh_password])
-      end
-    
-      def table_definition options
-        sql = "SELECT table_schema, table_name, column_name, is_nullable, data_type, column_type, column_key "
-        sql += "FROM INFORMATION_SCHEMA.COLUMNS where table_name = '#{options[:table]}' "
-        sql += "and table_schema = '#{options[:db]}';"
-      
-        puts sql
-      
-        desc = SourceDb.exec_sql(options[:db], sql)
-
-        return desc
-      end
-    end
-  end
-end
-

data/lib/loader/vertica/vertica_sql.rb~

@@ -1,60 +0,0 @@
-module VerticaUtils
-  class VerticaSql
-  
-    def self.create_table_stmt options
-
-      sql = "CREATE TABLE IF NOT EXISTS #{options[:vertica_db]}."
-      sql += "#{options[:vertica_schema]}." if options[:vertica_schema]
-      sql += "#{options[:table]} ("
-
-      index = 1
-      primary_set = false
-
-      options[:columns].each do |column|
-        sql += "\"#{column['column_name']}\" "
-
-        sql += data_type(column['data_type'], column['column_type'])
-        sql += " "
-
-        if column['column_key'] == "PRI"
-          sql += key(column['column_key']) + " " unless primary_set # set only one primary key
-          primary_set = true
-        end
-
-        sql += nullable(column['is_nullable'])
-        sql += " "
-
-        if index < options[:columns].size
-          sql += ", "
-        else
-          sql += ");"
-        end
-        index += 1
-      end
- 
-      puts sql
-    
-      return sql
-    end
-  
-    def self.nullable is_nullable
-      if is_nullable == "YES"
-        return "NULL"
-      elsif is_nullable == "NO"
-        return "NOT NULL"
-      end
-      return ""
-    end
- 
-    def self.data_type type, col_type
-      type = VerticaTypes.convert type, col_type
-      result = " #{type} "
-      return result
-    end
-
-    def self.key col_key
-      col_key = VerticaTypes.convert_key col_key
-      return  "#{col_key} "
-    end
-  end
-end

data/lib/transporter/parallelizer.rb~

@@ -1,5 +0,0 @@
-module Myreplicator
-  class Parallelizer
-    
-  end
-end

data/test/dummy/app/controllers/test_controller.rb~

@@ -1,8 +0,0 @@
-class TestController < ActionController::Base
-  protect_from_forgery
-
-  def index
-
-  end
-
-end

data/test/dummy/config/database.yml~

@@ -1,47 +0,0 @@
-# SQLite version 3.x
-#   gem install sqlite3
-#
-#   Ensure the SQLite 3 gem is defined in your Gemfile
-#   gem 'sqlite3'
-development:
-  adapter: mysql2
-  database: myreplicator
-  username: sas
-  password: sasan
-  host: 127.0.0.1
-
-okl_dev:
-  adapter: mysql2
-  database: okl_dev
-  username: sas
-  password: sasan
-  host: 127.0.0.1
-
-okl_test:
-  adapter: mysql2
-  database: okl_test
-  username: xbox
-  password: okl
-  host: 54.243.176.8
-
-myreplicator:
-  adapter: mysql2
-  database: myreplicator
-  username: sas
-  password: sasan
-  host: 127.0.0.1
-
-# Warning: The database defined as "test" will be erased and
-# re-generated from your development database when you run "rake".
-# Do not set this db to the same as development or production.
-test:
-  adapter: sqlite3
-  database: db/test.sqlite3
-  pool: 5
-  timeout: 5000
-
-production:
-  adapter: sqlite3
-  database: db/production.sqlite3
-  pool: 5
-  timeout: 5000

data/test/dummy/config/myreplicator.yml~

@@ -1,23 +0,0 @@
-myreplicator:
-  tmp_path: 
-  mysqldump: mysqldump
-  mysqlimport: mysqlimport
-  mysql: mysql
-  outfile_location: /tmp/myreplicator
-  escape_by: '"'
-  terminate_by: '\t'
-  enclosed_by: '"'
-  lines_terminate_by: '\n'
-
-okl_dev:
-  ssh_host: localhost
-  ssh_user: guest
-  ssh_password: sasanguest
-  ssh_tmp_dir: /home/guest/tmp
-  
-okl_test:
-  ssh_host: 54.243.176.8
-  ssh_user: ubuntu
-  ssh_db_host: 127.0.0.1
-  ssh_private_key: ~/.ssh/oklbi-team.pem
-  ssh_tmp_dir: /home/ubuntu/myreplicator_tmp

data/test/fixtures/myreplicator/vertica_exports.yml

@@ -1,11 +0,0 @@
-# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/Fixtures.html
-
-# This model initially had no columns defined.  If you add columns to the
-# model remove the '{}' from the fixture names and add the columns immediately
-# below each fixture, per the syntax in the comments below
-#
-one: {}
-# column: value
-#
-two: {}
-#  column: value