myreplicator 1.0.6 → 1.1.0

data/lib/loader/vertica/vertica_loader.rb

@@ -0,0 +1,235 @@
+module Myreplicator
+  class VerticaLoader
+    class << self
+
+      def create_table *args
+        options = args.extract_options!
+        columns = []
+        options[:mysql_schema].each(:as => :hash) do |row|
+          columns << row
+        end
+        options[:columns] = columns
+
+        sql = Myreplicator::VerticaSql.create_table_stmt options
+        puts sql
+        VerticaDb::Base.connection.execute sql
+      end
+
+      def destination_table_vertica options
+        sql = "select column_name, data_type From columns where 
+                 table_name = '#{options[:table]}' AND table_schema = '#{options[:destination_schema]}'"
+        puts sql
+        result = DB.exec_sql("vertica",sql)
+        return result     
+      end
+
+      ##
+      # Schema Change Algorithm
+      # Create temp table to load all data
+      # Load data 
+      # Drop table
+      # Rename table
+      ##
+      # rasing a concern: using the same schema or the tmp schema for the tmp table? Vertica doesn't lock the schema
+      def apply_schema_change options, temp_table
+        Myreplicator::VerticaLoader.create_table({:mysql_schema => options[:mysql_schema],
+                                                            :vertica_db => options[:vertica_db], 
+                                                            :vertica_schema => options[:vertica_schema],
+                                                            :table => temp_table,
+                                                            :mysql_table => options[:table]
+                                                          })
+      end
+
+      def full_load options, temp_table
+        export_id = options[:export_id]
+        new_options = prepare_options options
+        begin
+          a = Myreplicator::Export.find(export_id)
+          a.max_incremental_value = "0"
+          a.save!
+          a.export
+          b = Myreplicator::Transporter
+          b.perform
+          file = File.join(Myreplicator.app_root,"tmp", "myreplicator", "#{a.filename}")
+          `gunzip #{file}.gz`
+          new_options[:file] = file
+          new_options[:table] = temp_table
+          new_options[:schema] = options[:vertica_schema]
+          
+          vertica_copy new_options
+          
+          #drop
+          FileUtils.rm file
+          FileUtils.rm "#{file}.json"
+          sql = "DROP TABLE #{options[:vertica_db]}.#{options[:vertica_schema]}.#{options[:table]} CASCADE;"
+          VerticaDb::Base.connection.execute sql
+          #rename
+          sql = "ALTER TABLE #{options[:vertica_db]}.#{options[:vertica_schema]}.#{temp_table} RENAME TO #{options[:table]};"
+          VerticaDb::Base.connection.execute sql
+        rescue Exception => e
+          raise e.message 
+        ensure
+          
+        end
+      end
+
+      # def create_all_tables db
+      #   tables = Loader::SourceDb.get_tables(db)
+      #   sqls = {}
+      #   tables.each do |table|
+      #     puts "Creating #{db}.#{table}"
+      #     sql = "DROP TABLE IF EXISTS #{db}.#{table} CASCADE;"
+      #     VerticaDb::Base.connection.execute sql
+      #     sql = Loader::VerticaLoader.create_table(:vertica_db => "bidw",
+      #     :vertica_table => table,
+      #     :vertica_schema => db,
+      #     :table => table,
+      #     :db => db)
+      #     sqls["#{table}"] = sql
+      #     VerticaDb::Base.connection.execute sql
+      #   end
+      # end
+    
+      def prepare_options *args
+        options = args.extract_options!
+        vertica_options =ActiveRecord::Base.configurations["vertica"]
+        
+        result = options.clone
+        result.reverse_merge!(
+          :host => vertica_options["host"],
+          :user => vertica_options["username"],
+          :pass => vertica_options["password"],
+          :db   => vertica_options["database"],
+          :schema => options[:destination_schema],
+          :table => options[:table_name],
+          :file => options[:filepath],
+          :delimiter => "\t",
+          :null_value => "NULL",
+          :enclosed => ""
+        )
+        if !vertica_options["vsql"].blank?
+          result.reverse_merge!(
+            :vsql => vertica_options["vsql"]
+            )
+        else
+          result.reverse_merge!(
+            :vsql => "/opt/vertica/bin/vsql"
+          )
+        end
+        
+        return result  
+      end
+      
+      # Loader::VerticaLoader.load({:schema => "king", :table => "category_overview_data", :file => "tmp/vertica/category_overview_data.tsv", :null_value => "NULL"})
+      def load *args
+        options = args.extract_options!
+        #options = {:table => "app_csvs", :db => "public", :source_schema => "okl_dev"}
+        schema_check = Myreplicator::MysqlExporter.schema_changed?(:table => options[:table_name], 
+                                                     :destination_schema => options[:destination_schema], 
+                                                     :source_schema => options[:source_schema])
+        
+        #create a temp table
+        temp_table = "temp_" + options[:table_name] + DateTime.now.strftime('%Y%m%d_%H%M%S').to_s
+        ops = {:mysql_schema => schema_check[:mysql_schema],
+               :vertica_db => options[:db],
+               :vertica_schema => options[:destination_schema],
+               :table => options[:table_name],
+               :export_id => options[:export_id]
+        }
+        begin
+          if schema_check[:new]
+            create_table(ops)
+            apply_schema_change(ops, temp_table)
+            #vertica_copy options
+            full_load(ops, temp_table) 
+          elsif schema_check[:changed]
+            apply_schema_change(ops, temp_table)
+            full_load(ops, temp_table)
+          else
+            vertica_copy options
+          end
+        rescue Exception => e
+          raise e.message
+        end
+      end
+      
+      def vertica_copy * args
+        options = args.extract_options!
+        list_of_nulls =  ["0000-00-00"]
+        prepared_options = prepare_options options
+        if prepared_options[:file].blank?
+          raise "No input file"
+        end
+        
+        begin
+          process_file(:file => prepared_options[:file], :list_of_nulls => list_of_nulls, :null_value => prepared_options[:null_value])
+          cmd = get_vsql_command(prepared_options)
+          Kernel.p cmd
+          system(cmd)
+        rescue Exception => e
+          raise e.message
+        end
+      end
+        
+      def get_vsql_command prepared_options
+        file_extension = prepared_options[:file].split('.').last
+        file_handler = ""
+        file_handler = "GZIP" if file_extension == "gz" 
+        sql = "COPY #{prepared_options[:schema]}.#{prepared_options[:table]} FROM LOCAL \'#{prepared_options[:file]}\' #{file_handler} DELIMITER E\'#{prepared_options[:delimiter]}\' NULL as \'#{prepared_options[:null_value]}\' ENCLOSED BY \'#{prepared_options[:enclosed]}\' EXCEPTIONS 'load_exceptions.log';"
+        cmd = "#{prepared_options[:vsql]} -h #{prepared_options[:host]} -U #{prepared_options[:user]} -w #{prepared_options[:pass]} -d #{prepared_options[:db]} -c \"#{sql}\""
+        return cmd
+      end
+      
+      def process_file *args
+        ### replace the null values in the input file 
+        options = args.extract_options!
+        options[:file].blank? ? return : file = options[:file]
+        options[:list_of_nulls].blank? ? list_of_nulls = [] : list_of_nulls = options[:list_of_nulls]
+        options[:null_value].blank? ? null_value = "NULL" : null_value = options[:null_value]
+        
+        file_extension = file.split('.').last
+        case file_extension
+        when "tsv", "csv"
+          process_flat_file(file, list_of_nulls, null_value)
+        when "gz"
+          process_gzip_file(file, list_of_nulls, null_value)
+        else
+          raise "Un supported file extension"
+        end
+      end
+      
+      def replace_null(file, list_of_nulls, null_value = "NULL")
+        list_of_nulls.each do | value|
+          # special case for NULL MySQL datetime/date type but the column is defined NOT NULL
+          if value == '0000-00-00'
+            cmd1 = "sed -i 's/#{value}/1900-01-01/g' #{file}"
+            Kernel.p cmd1
+            system(cmd1)
+          else
+            cmd1 = "sed -i 's/#{value}/#{null_value}/g' #{file}"
+            Kernel.p cmd1
+            system(cmd1)
+          end
+        end
+      end
+      
+      def process_flat_file file, list_of_nulls, null_value 
+        # sed
+        replace_null(file, list_of_nulls, null_value)
+      end
+      
+      def process_gzip_file file, list_of_nulls, null_value
+        # unzip
+        cmd = "gunzip -f #{file} -c > tmp/temp.txt"
+        system(cmd)
+        # sed
+        replace_null(file, list_of_nulls, null_value)
+        # zip
+        cmd2 = "gzip tmp/temp.txt -c > #{file}"
+        system(cmd2)
+      end
+
+    end
+  end
+end
+

data/lib/loader/vertica/vertica_loader.rb~

@@ -0,0 +1,175 @@
+require 'vertica_utils/types'
+require 'vertica_utils/source_db'
+require 'vertica_utils/vertica_sql'
+
+module VerticaUtils
+  class VerticaLoader
+    class << self
+      def create_table *args
+        options = args.extract_options!
+        columns = []
+        table_definition(options).each(:as => :hash) do |row|
+          columns << row
+        end
+        options[:columns] = columns
+      
+        sql = VerticaUtils::VerticaSql.create_table_stmt options
+      end
+    
+      def create_all_tables db
+        tables = VerticaUtils::SourceDb.get_tables(db)
+        sqls = {}
+        tables.each do |table|
+          puts "Creating #{db}.#{table}"
+          sql = "DROP TABLE IF EXISTS #{db}.#{table} CASCADE;"
+          VerticaDb::Base.connection.execute sql
+          sql = VerticaUtils::VerticaLoader.create_table(:vertica_db => "bidw",
+          :vertica_table => table,
+          :vertica_schema => db,
+          :table => table,
+          :db => db)
+          sqls["#{table}"] = sql
+          VerticaDb::Base.connection.execute sql
+        end
+      end
+    
+      def prepare_options *args
+        options = args.extract_options!
+        options.reverse_merge!(
+          :host => "sfo-load-dw-01",
+          :user => "vertica",
+          :pass => "test",
+          :db => "bidw",
+          :schema => "test",
+          :table => "",
+          :file => "",
+          :delimiter => "\t",
+          :null_value => "NULL",
+          :enclosed => ""
+        )
+        return options  
+      end
+      
+      # VerticaUtils::VerticaLoader.load_to_vertica({:schema => "king", :table => "category_overview_data", :file => "tmp/vertica/category_overview_data.tsv", :null_value => "NULL"})
+      def load_to_vertica *args
+        list_of_nulls =  ["0000-00-00"]
+        options = args.extract_options!
+        prepared_options = prepare_options options
+        Kernel.p prepared_options[:file]
+        if prepared_options[:file].blank?
+          raise "No input file"
+        end
+      
+        begin
+          process_file(:file => prepared_options[:file], :list_of_nulls => list_of_nulls, :null_value => prepared_options[:null_value])
+          cmd = get_vsql_command(prepared_options)
+          Kernel.p cmd
+          system(cmd)
+        rescue Exception => e
+          raise e.message
+        end
+      end
+        
+      def get_vsql_command prepared_options
+        file_extension = prepared_options[:file].split('.').last
+        file_handler = ""
+        file_handler = "GZIP" if file_extension == "gz" 
+        sql = "COPY #{prepared_options[:schema]}.#{prepared_options[:table]} FROM LOCAL \'#{prepared_options[:file]}\' #{file_handler} DELIMITER E\'#{prepared_options[:delimiter]}\' NULL as \'#{prepared_options[:null_value]}\' ENCLOSED BY \'#{prepared_options[:enclosed]}\' EXCEPTIONS 'tmp/vertica/load_exceptions.log';"
+        cmd = "/opt/vertica/bin/vsql -h #{prepared_options[:host]} -U #{prepared_options[:user]} -w #{prepared_options[:pass]} -d #{prepared_options[:db]} -c \"#{sql}\""
+        return cmd
+      end
+      
+      def process_file *args
+        ### replace the null values in the input file 
+        options = args.extract_options!
+        options[:file].blank? ? return : file = options[:file]
+        options[:list_of_nulls].blank? ? list_of_nulls = [] : list_of_nulls = options[:list_of_nulls]
+        options[:null_value].blank? ? null_value = "NULL" : null_value = options[:null_value]
+        
+        file_extension = file.split('.').last
+        case file_extension
+        when "tsv", "csv"
+          process_flat_file(file, list_of_nulls, null_value)
+        when "gz"
+          process_gzip_file(file, list_of_nulls, null_value)
+        else
+          raise "Un supported file extension"
+        end
+      end
+      
+      def replace_null(file, list_of_nulls, null_value = "NULL")
+        list_of_nulls.each do | value|
+          # special case for NULL MySQL datetime/date type but the column is defined NOT NULL
+          if value == '0000-00-00'
+            cmd1 = "sed -i 's/#{value}/1900-01-01/g' #{file}"
+            Kernel.p cmd1
+            system(cmd1)
+          else
+            cmd1 = "sed -i 's/#{value}/#{null_value}/g' #{file}"
+            Kernel.p cmd1
+            system(cmd1)
+          end
+        end
+      end
+      
+      def process_flat_file file, list_of_nulls, null_value 
+        # sed
+        replace_null(file, list_of_nulls, null_value)
+      end
+      
+      def process_gzip_file file, list_of_nulls, null_value
+        # unzip
+        cmd = "gunzip -f #{file} -c > tmp/temp.txt"
+        system(cmd)
+        # sed
+        replace_null(file, list_of_nulls, null_value)
+        # zip
+        cmd2 = "gzip tmp/temp.txt -c > #{file}"
+        system(cmd2)
+      end
+        
+      def get_table_columns * args
+        options = args.extract_options!
+        if options[:host].blank? or options[:user].blank?  or options[:pass].blank? or options[:db].blank? or options[:schema].blank? or options[:table].blank? 
+          raise "Unspecified host, user, pass, db, schema or table"
+        end
+        result = {}
+        count = 0
+        begin
+          cmd = "/opt/vertica/bin/vsql -h #{options[:host]} -U #{options[:user]} -w #{options[:pass]} -d #{options[:db]} -c '\\d #{options[:schema]}.#{options[:table]}';"
+          puts cmd
+          output = `#{cmd}`
+          lines = output.split("\n")
+          data = lines[3..lines.size-2]
+          data.each do |item|
+            count += 1
+            column_name = item.split("|")[2].lstrip!.rstrip!
+            column_type = item.split("|")[3].lstrip!.rstrip!
+            puts column_name + " " + column_type
+            result[count] = {column_name => column_type}
+          end
+        rescue Exception => e
+          raise e.message
+        end
+        return result
+      end
+        
+      def ssh_connection options
+        ssh = Net::SSH.start(options[:ssh_host], options[:ssh_user], :password => options[:ssh_password])
+      end
+    
+      def table_definition options
+        sql = "SELECT table_schema, table_name, column_name, is_nullable, data_type, column_type, column_key "
+        sql += "FROM INFORMATION_SCHEMA.COLUMNS where table_name = '#{options[:table]}' "
+        sql += "and table_schema = '#{options[:db]}';"
+      
+        puts sql
+      
+        desc = SourceDb.exec_sql(options[:db], sql)
+
+        return desc
+      end
+    end
+  end
+end
+

data/lib/loader/vertica/vertica_sql.rb

@@ -0,0 +1,59 @@
+module Myreplicator
+  class VerticaSql
+  
+    def self.create_table_stmt options
+      sql = "CREATE TABLE IF NOT EXISTS #{options[:vertica_db]}."
+      sql += "#{options[:vertica_schema]}." if options[:vertica_schema]
+      sql += "#{options[:table]} ("
+
+      index = 1
+      primary_set = false
+
+      options[:columns].each do |column|
+        sql += "\"#{column['column_name']}\" "
+
+        sql += data_type(column['data_type'], column['column_type'])
+        sql += " "
+
+        if column['column_key'] == "PRI"
+          sql += key(column['column_key']) + " " unless primary_set # set only one primary key
+          primary_set = true
+        end
+
+        sql += nullable(column['is_nullable'])
+        sql += " "
+
+        if index < options[:columns].size
+          sql += ", "
+        else
+          sql += ");"
+        end
+        index += 1
+      end
+ 
+      puts sql
+    
+      return sql
+    end
+  
+    def self.nullable is_nullable
+      if is_nullable == "YES"
+        return "NULL"
+      elsif is_nullable == "NO"
+        return "NOT NULL"
+      end
+      return ""
+    end
+ 
+    def self.data_type type, col_type
+      type = VerticaTypes.convert type, col_type
+      result = " #{type} "
+      return result
+    end
+
+    def self.key col_key
+      col_key = VerticaTypes.convert_key col_key
+      return  "#{col_key} "
+    end
+  end
+end

data/lib/loader/vertica/vertica_sql.rb~

@@ -0,0 +1,60 @@
+module VerticaUtils
+  class VerticaSql
+  
+    def self.create_table_stmt options
+
+      sql = "CREATE TABLE IF NOT EXISTS #{options[:vertica_db]}."
+      sql += "#{options[:vertica_schema]}." if options[:vertica_schema]
+      sql += "#{options[:table]} ("
+
+      index = 1
+      primary_set = false
+
+      options[:columns].each do |column|
+        sql += "\"#{column['column_name']}\" "
+
+        sql += data_type(column['data_type'], column['column_type'])
+        sql += " "
+
+        if column['column_key'] == "PRI"
+          sql += key(column['column_key']) + " " unless primary_set # set only one primary key
+          primary_set = true
+        end
+
+        sql += nullable(column['is_nullable'])
+        sql += " "
+
+        if index < options[:columns].size
+          sql += ", "
+        else
+          sql += ");"
+        end
+        index += 1
+      end
+ 
+      puts sql
+    
+      return sql
+    end
+  
+    def self.nullable is_nullable
+      if is_nullable == "YES"
+        return "NULL"
+      elsif is_nullable == "NO"
+        return "NOT NULL"
+      end
+      return ""
+    end
+ 
+    def self.data_type type, col_type
+      type = VerticaTypes.convert type, col_type
+      result = " #{type} "
+      return result
+    end
+
+    def self.key col_key
+      col_key = VerticaTypes.convert_key col_key
+      return  "#{col_key} "
+    end
+  end
+end

data/lib/loader/vertica.rb

@@ -0,0 +1 @@
+Dir["#{File.expand_path(File.dirname(__FILE__))}/vertica/*.rb"].each { | f | require File.expand_path(f) }

data/lib/myreplicator/version.rb

@@ -1,3 +1,3 @@
 module Myreplicator
-  VERSION = "1.0.6"
+  VERSION = "1.1.0"
 end

data/lib/transporter/transporter.rb

@@ -100,8 +100,13 @@ module Myreplicator
             Log.run(:job_type => "transporter", :name => "export_file",
                     :file => dump_file, :export_id => export.id) do |log|
               puts "Downloading #{dump_file}"
-              sftp.download!(dump_file, File.join(tmp_dir, dump_file.split("/").last))
+              local_dump_file = File.join(tmp_dir, dump_file.split("/").last)
+              sftp.download!(dump_file, local_dump_file)
               Transporter.remove!(export, json_file, dump_file)
+              # store back up as well
+              unless metadata.store_in.blank?
+                Transporter.backup_files(metadata.backup_path, json_local_path, local_dump_file)
+              end
             end
           end #if
           puts "#{Thread.current.to_s}___Exiting download..."
@@ -109,6 +114,11 @@ module Myreplicator
       }
     end
 
+    def self.backup_files location, metadata_path, dump_path
+      FileUtils.cp(metadata_path, location)
+      FileUtils.cp(dump_path, location)
+    end
+
     ##
     # Returns true if the file should be deleted
     ##

data/test/dummy/app/models/vertica_db.rb

@@ -0,0 +1,7 @@
+module VerticaDb
+  class Base < ActiveRecord::Base
+    establish_connection(ActiveRecord::Base.configurations["vertica"])
+  end
+end
+
+Dir['vertica_db/*.rb'].each { | f | require File.expand_path(f) }

data/test/dummy/config/database.yml

@@ -31,6 +31,13 @@ myreplicator:
   password: sasan
   host: 127.0.0.1
 
+vertica:
+  adapter: vertica
+  host: ec2-23-22-175-247.compute-1.amazonaws.com
+  username: jferris
+  password: okl
+  database: test
+
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
 # Do not set this db to the same as development or production.

data/test/dummy/config/database.yml~

@@ -24,9 +24,9 @@ okl_test:
   password: okl
   host: 54.243.176.8
 
-destination_db:
+myreplicator:
   adapter: mysql2
-  database: raw_sources
+  database: myreplicator
   username: sas
   password: sasan
   host: 127.0.0.1

data/test/dummy/config/myreplicator.yml

@@ -9,6 +9,11 @@ myreplicator:
   enclosed_by: '"'
   lines_terminate_by: '\n'
 
+  ssh_host: localhost
+  ssh_user: guest
+  ssh_password: sasanguest
+  ssh_tmp_dir: /home/guest/tmp
+
 okl_dev:
   ssh_host: localhost
   ssh_user: guest

data/test/dummy/config/myreplicator.yml~

@@ -8,8 +8,6 @@ myreplicator:
   terminate_by: '\t'
   enclosed_by: '"'
   lines_terminate_by: '\n'
-  auth_required: yes
-  login_redirect: /
 
 okl_dev:
   ssh_host: localhost