myreplicator 1.1.5 → 1.1.6

data/db/migrate/20130213211927_create_myreplicator_vertica_exports.rb~

@@ -0,0 +1,8 @@
+class CreateMyreplicatorVerticaExports < ActiveRecord::Migration
+  def change
+    create_table :myreplicator_vertica_exports do |t|
+
+      t.timestamps
+    end
+  end
+end

data/lib/configuration.rb~

@@ -0,0 +1,23 @@
+module Myreplicator
+  ##
+  # Configuration class for avoiding load the yml file everytime and cleaner config settings
+  ##
+  class Configuration
+    class << self; attr_accessor :tmp_path, :mysqldump, :mysql end
+    
+    # yml = YAML.load(File.read("#{Myreplicator.app_root}/config/myreplicator.yml"))
+    # Kernel.p yml["myreplicator"]
+
+    # @@tmp_path = yml["myreplicator"]["tmp_path"]
+    # @@mysql = yml["myreplicator"]["mysql"]
+    # @@mysqldump = yml["myreplicator"]["mysqldump"]
+  end
+
+  def self.config(&block)
+    @@config ||= Myreplicator::Configuration.new
+
+    yield @@config if block
+
+    return @@config
+  end
+end

data/lib/exporter/mysql_exporter.rb~

@@ -0,0 +1,6 @@
+module Myreplicator
+  class MysqlExporter
+    
+    
+  end
+end

data/lib/exporter/sql_commands.rb

@@ -138,7 +138,7 @@ module Myreplicator
 
     ##
     # Export using outfile
-    # ;~; delimited
+    # \\0 delimited
     # terminated by newline 
     # Location of the output file needs to have 777 perms
     ##
@@ -148,10 +148,16 @@ module Myreplicator
       db = options[:db]
 
       # Database host when ssh'ed into the db server
-      db_host = ssh_configs(db)["ssh_db_host"].nil? ? "127.0.0.1" : ssh_configs(db)["ssh_db_host"]
-
+      db_host = "127.0.0.1"
+      
+      if !ssh_configs(db)["ssh_db_host"].blank?
+        db_host =  ssh_configs(db)["ssh_db_host"]
+      elsif !db_configs(db)["host"].blank?
+        db_host = db_configs(db)["host"]
+      end
+      
       flags = ""
-
+      
       self.mysql_flags.each_pair do |flag, value|
         if options[:flags].include? flag
           flags += " --#{flag} "
@@ -160,21 +166,20 @@ module Myreplicator
         end
       end
 
-        # Database host when ssh'ed into the db server
-        db_host = "127.0.0.1"
-        
-        if !ssh_configs(db)["ssh_db_host"].blank?
-          db_host =  ssh_configs(db)["ssh_db_host"]
-        elsif !db_configs(db)["host"].blank?
-          db_host = db_configs(db)["host"]
-        end
-        
-        cmd = Myreplicator.mysql
-        cmd += "#{flags} -u#{db_configs(db)["username"]} -p#{db_configs(db)["password"]} "
-        cmd += "-h#{db_host} "
-        cmd += " -P#{db_configs(db)["port"]} " if db_configs(db)["port"]
-        cmd += "--execute=\"#{get_outfile_sql(options)}\" "
-        puts cmd
+      cmd = Myreplicator.mysql
+      cmd += "#{flags} "
+      
+      if db_configs(db).has_key? "socket"
+        cmd += "--socket=#{db_configs(db)["socket"]} " 
+      else
+        cmd += "-u#{db_configs(db)["username"]} -p#{db_configs(db)["password"]} " 
+      end
+      
+      cmd += "-h#{db_host} " if db_configs(db)["host"].blank?
+      cmd += db_configs(db)["port"].blank? ? "-P3306 " : "-P#{db_configs(db)["port"]} "
+      cmd += "--execute=\"#{get_outfile_sql(options)}\" "
+      
+      puts cmd
       return cmd
     end
 

data/lib/exporter.rb~

@@ -0,0 +1 @@
+require ""

data/lib/loader/vertica/source_db.rb~

@@ -0,0 +1,30 @@
+module VerticaUtils
+  class SourceDb < ActiveRecord::Base
+      
+    def self.connect db
+      establish_connection(ActiveRecord::Base.configurations[db])
+    end
+  
+    ##
+    # Returns tables as an Array
+    # releases the connection
+    ##
+    def self.get_tables(db)
+      tables = []
+      begin
+        self.connect(db)
+        tables = self.connection.tables
+        self.connection_pool.release_connection
+      rescue Mysql2::Error => e
+        puts "Connection to #{db} Failed!"
+        puts e.message
+      end
+      return tables
+    end
+  
+    def self.exec_sql source_db,sql
+      SourceDb.connect(source_db)
+      return SourceDb.connection.execute(sql)
+    end
+  end
+end

data/lib/loader/vertica/vertica_loader.rb~

@@ -0,0 +1,175 @@
+require 'vertica_utils/types'
+require 'vertica_utils/source_db'
+require 'vertica_utils/vertica_sql'
+
+module VerticaUtils
+  class VerticaLoader
+    class << self
+      def create_table *args
+        options = args.extract_options!
+        columns = []
+        table_definition(options).each(:as => :hash) do |row|
+          columns << row
+        end
+        options[:columns] = columns
+      
+        sql = VerticaUtils::VerticaSql.create_table_stmt options
+      end
+    
+      def create_all_tables db
+        tables = VerticaUtils::SourceDb.get_tables(db)
+        sqls = {}
+        tables.each do |table|
+          puts "Creating #{db}.#{table}"
+          sql = "DROP TABLE IF EXISTS #{db}.#{table} CASCADE;"
+          VerticaDb::Base.connection.execute sql
+          sql = VerticaUtils::VerticaLoader.create_table(:vertica_db => "bidw",
+          :vertica_table => table,
+          :vertica_schema => db,
+          :table => table,
+          :db => db)
+          sqls["#{table}"] = sql
+          VerticaDb::Base.connection.execute sql
+        end
+      end
+    
+      def prepare_options *args
+        options = args.extract_options!
+        options.reverse_merge!(
+          :host => "sfo-load-dw-01",
+          :user => "vertica",
+          :pass => "test",
+          :db => "bidw",
+          :schema => "test",
+          :table => "",
+          :file => "",
+          :delimiter => "\t",
+          :null_value => "NULL",
+          :enclosed => ""
+        )
+        return options  
+      end
+      
+      # VerticaUtils::VerticaLoader.load_to_vertica({:schema => "king", :table => "category_overview_data", :file => "tmp/vertica/category_overview_data.tsv", :null_value => "NULL"})
+      def load_to_vertica *args
+        list_of_nulls =  ["0000-00-00"]
+        options = args.extract_options!
+        prepared_options = prepare_options options
+        Kernel.p prepared_options[:file]
+        if prepared_options[:file].blank?
+          raise "No input file"
+        end
+      
+        begin
+          process_file(:file => prepared_options[:file], :list_of_nulls => list_of_nulls, :null_value => prepared_options[:null_value])
+          cmd = get_vsql_command(prepared_options)
+          Kernel.p cmd
+          system(cmd)
+        rescue Exception => e
+          raise e.message
+        end
+      end
+        
+      def get_vsql_command prepared_options
+        file_extension = prepared_options[:file].split('.').last
+        file_handler = ""
+        file_handler = "GZIP" if file_extension == "gz" 
+        sql = "COPY #{prepared_options[:schema]}.#{prepared_options[:table]} FROM LOCAL \'#{prepared_options[:file]}\' #{file_handler} DELIMITER E\'#{prepared_options[:delimiter]}\' NULL as \'#{prepared_options[:null_value]}\' ENCLOSED BY \'#{prepared_options[:enclosed]}\' EXCEPTIONS 'tmp/vertica/load_exceptions.log';"
+        cmd = "/opt/vertica/bin/vsql -h #{prepared_options[:host]} -U #{prepared_options[:user]} -w #{prepared_options[:pass]} -d #{prepared_options[:db]} -c \"#{sql}\""
+        return cmd
+      end
+      
+      def process_file *args
+        ### replace the null values in the input file 
+        options = args.extract_options!
+        options[:file].blank? ? return : file = options[:file]
+        options[:list_of_nulls].blank? ? list_of_nulls = [] : list_of_nulls = options[:list_of_nulls]
+        options[:null_value].blank? ? null_value = "NULL" : null_value = options[:null_value]
+        
+        file_extension = file.split('.').last
+        case file_extension
+        when "tsv", "csv"
+          process_flat_file(file, list_of_nulls, null_value)
+        when "gz"
+          process_gzip_file(file, list_of_nulls, null_value)
+        else
+          raise "Un supported file extension"
+        end
+      end
+      
+      def replace_null(file, list_of_nulls, null_value = "NULL")
+        list_of_nulls.each do | value|
+          # special case for NULL MySQL datetime/date type but the column is defined NOT NULL
+          if value == '0000-00-00'
+            cmd1 = "sed -i 's/#{value}/1900-01-01/g' #{file}"
+            Kernel.p cmd1
+            system(cmd1)
+          else
+            cmd1 = "sed -i 's/#{value}/#{null_value}/g' #{file}"
+            Kernel.p cmd1
+            system(cmd1)
+          end
+        end
+      end
+      
+      def process_flat_file file, list_of_nulls, null_value 
+        # sed
+        replace_null(file, list_of_nulls, null_value)
+      end
+      
+      def process_gzip_file file, list_of_nulls, null_value
+        # unzip
+        cmd = "gunzip -f #{file} -c > tmp/temp.txt"
+        system(cmd)
+        # sed
+        replace_null(file, list_of_nulls, null_value)
+        # zip
+        cmd2 = "gzip tmp/temp.txt -c > #{file}"
+        system(cmd2)
+      end
+        
+      def get_table_columns * args
+        options = args.extract_options!
+        if options[:host].blank? or options[:user].blank?  or options[:pass].blank? or options[:db].blank? or options[:schema].blank? or options[:table].blank? 
+          raise "Unspecified host, user, pass, db, schema or table"
+        end
+        result = {}
+        count = 0
+        begin
+          cmd = "/opt/vertica/bin/vsql -h #{options[:host]} -U #{options[:user]} -w #{options[:pass]} -d #{options[:db]} -c '\\d #{options[:schema]}.#{options[:table]}';"
+          puts cmd
+          output = `#{cmd}`
+          lines = output.split("\n")
+          data = lines[3..lines.size-2]
+          data.each do |item|
+            count += 1
+            column_name = item.split("|")[2].lstrip!.rstrip!
+            column_type = item.split("|")[3].lstrip!.rstrip!
+            puts column_name + " " + column_type
+            result[count] = {column_name => column_type}
+          end
+        rescue Exception => e
+          raise e.message
+        end
+        return result
+      end
+        
+      def ssh_connection options
+        ssh = Net::SSH.start(options[:ssh_host], options[:ssh_user], :password => options[:ssh_password])
+      end
+    
+      def table_definition options
+        sql = "SELECT table_schema, table_name, column_name, is_nullable, data_type, column_type, column_key "
+        sql += "FROM INFORMATION_SCHEMA.COLUMNS where table_name = '#{options[:table]}' "
+        sql += "and table_schema = '#{options[:db]}';"
+      
+        puts sql
+      
+        desc = SourceDb.exec_sql(options[:db], sql)
+
+        return desc
+      end
+    end
+  end
+end
+

data/lib/loader/vertica/vertica_sql.rb~

@@ -0,0 +1,60 @@
+module VerticaUtils
+  class VerticaSql
+  
+    def self.create_table_stmt options
+
+      sql = "CREATE TABLE IF NOT EXISTS #{options[:vertica_db]}."
+      sql += "#{options[:vertica_schema]}." if options[:vertica_schema]
+      sql += "#{options[:table]} ("
+
+      index = 1
+      primary_set = false
+
+      options[:columns].each do |column|
+        sql += "\"#{column['column_name']}\" "
+
+        sql += data_type(column['data_type'], column['column_type'])
+        sql += " "
+
+        if column['column_key'] == "PRI"
+          sql += key(column['column_key']) + " " unless primary_set # set only one primary key
+          primary_set = true
+        end
+
+        sql += nullable(column['is_nullable'])
+        sql += " "
+
+        if index < options[:columns].size
+          sql += ", "
+        else
+          sql += ");"
+        end
+        index += 1
+      end
+ 
+      puts sql
+    
+      return sql
+    end
+  
+    def self.nullable is_nullable
+      if is_nullable == "YES"
+        return "NULL"
+      elsif is_nullable == "NO"
+        return "NOT NULL"
+      end
+      return ""
+    end
+ 
+    def self.data_type type, col_type
+      type = VerticaTypes.convert type, col_type
+      result = " #{type} "
+      return result
+    end
+
+    def self.key col_key
+      col_key = VerticaTypes.convert_key col_key
+      return  "#{col_key} "
+    end
+  end
+end

data/lib/myreplicator/version.rb

@@ -1,3 +1,3 @@
 module Myreplicator
-  VERSION = "1.1.5"
+  VERSION = "1.1.6"
 end

data/lib/transporter/parallelizer.rb~

@@ -0,0 +1,5 @@
+module Myreplicator
+  class Parallelizer
+    
+  end
+end

data/test/dummy/app/controllers/test_controller.rb~

@@ -0,0 +1,8 @@
+class TestController < ActionController::Base
+  protect_from_forgery
+
+  def index
+
+  end
+
+end

data/test/dummy/config/database.yml

@@ -1,20 +1,13 @@
 # SQLite version 3.x
-# gem install sqlite3
+#   gem install sqlite3
 #
-# Ensure the SQLite 3 gem is defined in your Gemfile
-# gem 'sqlite3'
-vertica:
-  adapter: vertica
-  database: test
-  username: jferris
-  password: okl
-  host: ec2-23-22-175-247.compute-1.amazonaws.com
-  vsql: /Users/lnguyen/opt/vertica/bin/vsql
+#   Ensure the SQLite 3 gem is defined in your Gemfile
+#   gem 'sqlite3'
 development:
   adapter: mysql2
   database: myreplicator
-  username:
-  password:
+  username: sas
+  password: sasan
   host: 127.0.0.1
 
 okl_dev:
@@ -23,6 +16,7 @@ okl_dev:
   username: sas
   password: sasan
   host: 127.0.0.1
+  socket: /home/sasan/mysql.sock
   myreplicator: true
 
 okl_test:
@@ -40,6 +34,13 @@ myreplicator:
   password: sasan
   host: 127.0.0.1
 
+vertica:
+  adapter: vertica
+  host: ec2-23-22-175-247.compute-1.amazonaws.com
+  username: jferris
+  password: okl
+  database: test
+
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
 # Do not set this db to the same as development or production.
@@ -53,4 +54,4 @@ production:
   adapter: sqlite3
   database: db/production.sqlite3
   pool: 5
-  timeout: 5000
+  timeout: 5000

data/test/dummy/config/database.yml~

@@ -0,0 +1,56 @@
+# SQLite version 3.x
+#   gem install sqlite3
+#
+#   Ensure the SQLite 3 gem is defined in your Gemfile
+#   gem 'sqlite3'
+development:
+  adapter: mysql2
+  database: myreplicator
+  username: sas
+  password: sasan
+  host: 127.0.0.1
+
+okl_dev:
+  adapter: mysql2
+  database: okl_dev
+  username: sas
+  password: sasan
+  host: 127.0.0.1
+  myreplicator: true
+
+okl_test:
+  adapter: mysql2
+  database: okl_test
+  username: xbox
+  password: okl
+  host: 54.243.176.8
+  myreplicator: true
+
+myreplicator:
+  adapter: mysql2
+  database: myreplicator
+  username: sas
+  password: sasan
+  host: 127.0.0.1
+
+vertica:
+  adapter: vertica
+  host: ec2-23-22-175-247.compute-1.amazonaws.com
+  username: jferris
+  password: okl
+  database: test
+
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test:
+  adapter: sqlite3
+  database: db/test.sqlite3
+  pool: 5
+  timeout: 5000
+
+production:
+  adapter: sqlite3
+  database: db/production.sqlite3
+  pool: 5
+  timeout: 5000

data/test/dummy/config/myreplicator.yml

@@ -1,24 +1,28 @@
 myreplicator:
   tmp_path: 
-  mysqldump: /usr/local/mysql/bin/mysqldump
-  mysqlimport: /usr/local/mysql/bin/mysqlimport
-  mysql: /usr/local/mysql/bin/mysql
+  mysqldump: mysqldump
+  mysqlimport: mysqlimport
+  mysql: mysql
   outfile_location: /tmp/myreplicator
   escape_by: '"'
   terminate_by: '\t'
   enclosed_by: '"'
   lines_terminate_by: '\n'
-  login_redirect: /
 
-okl_dev:
   ssh_host: localhost
-  ssh_user: lnguyen
-  ssh_password: 'H!13lPa:'
-  ssh_tmp_dir: /Users/lnguyen/tmp
+  ssh_user: guest
+  ssh_password: sasanguest
+  ssh_tmp_dir: /home/guest/tmp
 
+okl_dev:
+  ssh_host: localhost
+  ssh_user: guest
+  ssh_password: sasanguest
+  ssh_tmp_dir: /home/guest/tmp
+  
 okl_test:
   ssh_host: 54.243.176.8
   ssh_user: ubuntu
   ssh_db_host: 127.0.0.1
-  ssh_private_key: ~/oklbi-team.pem
-  ssh_tmp_dir: /tmp
+  ssh_private_key: ~/.ssh/oklbi-team.pem
+  ssh_tmp_dir: /home/ubuntu/myreplicator_tmp

data/test/dummy/config/myreplicator.yml~

@@ -0,0 +1,23 @@
+myreplicator:
+  tmp_path: 
+  mysqldump: mysqldump
+  mysqlimport: mysqlimport
+  mysql: mysql
+  outfile_location: /tmp/myreplicator
+  escape_by: '"'
+  terminate_by: '\t'
+  enclosed_by: '"'
+  lines_terminate_by: '\n'
+
+okl_dev:
+  ssh_host: localhost
+  ssh_user: guest
+  ssh_password: sasanguest
+  ssh_tmp_dir: /home/guest/tmp
+  
+okl_test:
+  ssh_host: 54.243.176.8
+  ssh_user: ubuntu
+  ssh_db_host: 127.0.0.1
+  ssh_private_key: ~/.ssh/oklbi-team.pem
+  ssh_tmp_dir: /home/ubuntu/myreplicator_tmp