myreplicator 1.1.7 → 1.1.9

data/app/controllers/myreplicator/exports_controller.rb

@@ -45,8 +45,6 @@ module Myreplicator
       @dbs = get_dbs
       @tables = db_metadata
       @edit = true
-
-      Myreplicator::Export.schedule_in_resque # schedule in resque
     end
   
     # POST /exports
@@ -70,14 +68,13 @@ module Myreplicator
     # PUT /exports/1.json
     def update
       @export = Export.find(params[:id])
-      Myreplicator::Export.schedule_in_resque # schedule in resque
-
       @dbs = get_dbs
   
       respond_to do |format|
         if @export.update_attributes(params[:export])
           format.html { redirect_to @export, notice: 'Export was successfully updated.' }
           format.json { head :no_content }
+          Myreplicator::Export.schedule_in_resque # schedule in resque
         else
           format.html { render action: "edit" }
           format.json { render json: @export.errors, status: :unprocessable_entity }

data/app/models/myreplicator/log.rb

@@ -81,7 +81,7 @@ module Myreplicator
         logs.each do |log|
           begin
             Process.getpgid(log.pid)
-            puts "still running #{filepath}"
+            puts "still running #{log.file}"
             return true
           rescue Errno::ESRCH
             log.mark_dead

data/lib/exporter/export_metadata.rb

@@ -39,6 +39,16 @@ module Myreplicator
       name = zipped ? "#{name}.gz" : name
       return name     
     end
+    
+    def metadata_filename
+      name = filepath.split("/").last
+      name += ".json"
+      return name
+    end
+
+    def metadata_filepath tmp_dir
+      File.join(tmp_dir, metadata_filename)
+    end
 
     def destination_filepath tmp_dir
       File.join(tmp_dir, filename)

data/lib/exporter/mysql_exporter.rb

@@ -27,7 +27,7 @@ module Myreplicator
           on_failure_state_trans(metadata, "new") # If failed, go back to new
           on_export_success(metadata)
           initial_export metadata
-        elsif @export_obj.export_type? == :incremental or load_to == "vertica"
+        elsif @export_obj.export_type? == :incremental || load_to == "vertica"
           on_failure_state_trans(metadata, "failed") # Set state trans on failure
           on_export_success(metadata)
           incremental_export_into_outfile metadata
@@ -100,43 +100,6 @@ module Myreplicator
       return cmd
     end
     
-    ##
-    # Exports table incrementally, using the incremental column specified
-    # If column is not specified, it will export the entire table 
-    # Maximum value of the incremental column is recorded BEFORE export starts
-    ##
-
-    def incremental_export metadata
-      unless @export_obj.is_running?
-        max_value = @export_obj.max_value
-        metadata.export_type = "incremental"
-        @export_obj.update_max_val if @export_obj.max_incremental_value.blank?   
-        
-        cmd = incremental_export_cmd 
-        exporting_state_trans # mark exporting
-        puts "Exporting..."
-        result = execute_export(cmd, metadata)
-        check_result(result, 0)
-        metadata.incremental_val = max_value # store max val in metadata
-        @export_obj.update_max_val(max_value) # update max value if export was successful
-      end
-      return false
-    end
-
-    def incremental_export_cmd
-      sql = SqlCommands.export_sql(:db => @export_obj.source_schema,
-                                   :table => @export_obj.table_name,
-                                   :incremental_col => @export_obj.incremental_column,
-                                   :incremental_col_type => @export_obj.incremental_column_type,
-                                   :incremental_val => @export_obj.max_incremental_value)      
-      
-      cmd = SqlCommands.mysql_export(:db => @export_obj.source_schema,
-                                     :filepath => filepath,
-                                     :sql => sql)
-      
-      return cmd
-    end
-
     ##
     # Exports table incrementally, similar to incremental_export method
     # Dumps file in tmp directory specified in myreplicator.yml
@@ -146,25 +109,33 @@ module Myreplicator
 
     def incremental_export_into_outfile metadata
       unless @export_obj.is_running?
-
+        
         if @export_obj.export_type == "incremental"
           max_value = @export_obj.max_value
-          metadata.export_type = "incremental_outfile"        
+          metadata.export_type = "incremental"        
           @export_obj.update_max_val if @export_obj.max_incremental_value.blank?   
         end
 
         options = {
           :db => @export_obj.source_schema,
+          :source_schema => @export_obj.source_schema,
           :table => @export_obj.table_name,
           :filepath => filepath,
           :destination_schema => @export_obj.destination_schema}
 
-        unless MysqlExporter.schema_changed?(options)[:changed]
+        schema_status = MysqlExporter.schema_changed?(options)
+        Kernel.p "===== schema_status ====="
+        Kernel.p schema_status
+        if schema_status[:changed] # && new?
+          metadata.export_type = "initial"
+        else
           options[:incremental_col] = @export_obj.incremental_column
           options[:incremental_col_type] = @export_obj.incremental_column_type
           options[:incremental_val] = @export_obj.max_incremental_value
         end
 
+        Kernel.p "===== incremental_export_into_outfile OPTIONS ====="
+        Kernel.p options
         cmd = SqlCommands.mysql_export_outfile(options)      
         exporting_state_trans
         puts "Exporting..."
@@ -181,11 +152,10 @@ module Myreplicator
     end
 
     def self.compare_schemas vertica_schema, mysql_schema
+      Kernel.p vertica_schema
+      Kernel.p mysql_schema
       if vertica_schema.size != mysql_schema.size
-        return {:changed => true, 
-          :mysql_schema => mysql_schema, 
-          :vertica_schema => vertica_schema,
-          :new => false}
+        return true
       else
         index = 0
         while index < vertica_schema.size
@@ -195,7 +165,7 @@ module Myreplicator
           end
   
           # check for column's data type
-          if (vertica_schema.rows[index][:data_type] != VerticaTypes.convert(mysql_schema[index]["data_type"],mysql_schema[index]["column_type"]) and vertica_schema.rows[index][:data_type] != "timestamp")
+          if compare_datatypes index, vertica_schema, mysql_schema
             return true
           end
           # and others ?? (PRIMARY, DEFAULT NULL, etc.)
@@ -205,7 +175,28 @@ module Myreplicator
       return false      
     end
 
+    def self.compare_datatypes index, vertica_schema, mysql_schema
+      type = VerticaTypes.convert mysql_schema[index]["data_type"], mysql_schema[index]["column_type"]
+      if vertica_schema.rows[index][:data_type] != type
+        if vertica_schema.rows[index][:data_type] != "timestamp"
+          return true
+        end
+        return false
+      end
+      return false
+    end
+    
+    def self.get_mysql_schema_rows mysql_schema 
+      mysql_schema_simple_form = []
+      mysql_schema.each(:as => :hash) do |row|
+        puts row
+        mysql_schema_simple_form << row
+      end
+      return mysql_schema_simple_form
+    end
+
     def self.schema_changed? options
+      Kernel.p "===== schema_changed? ====="
       puts options
       mysql_schema = Loader.mysql_table_definition(options)
       vertica_schema = VerticaLoader.destination_table_vertica(options)
@@ -214,19 +205,19 @@ module Myreplicator
       unless vertica_schema.size > 0 
         return {:changed => true, :mysql_schema => mysql_schema, :new => true}
       end
-      
+      Kernel.p "***** 1 *****"
       # compare two schemas
-      mysql_schema_simple_form = []
-      mysql_schema.each(:as => :hash) do |row|
-        mysql_schema_simple_form << row  
-      end
       
-      mysql_schema_2 = mysql_schema_simple_form
+      
+      mysql_schema_2 = get_mysql_schema_rows mysql_schema
       if compare_schemas(vertica_schema, mysql_schema_2)
+        Kernel.p "***** 2 *****"
         result =  {:changed => true, :mysql_schema => mysql_schema, :vertica_schema => vertica_schema,:new => false}
       else
-        result =  {:changed => false}
+        Kernel.p "***** 3 *****"
+        result =  {:changed => false, :mysql_schema => mysql_schema}
       end
+      Kernel.p result
       return result
     end
 

data/lib/exporter/sql_commands.rb

@@ -119,6 +119,8 @@ module Myreplicator
     ##
     
     def self.get_outfile_sql options 
+      Kernel.p "===== SELECT * INTO OUTFILE OPTIONS====="
+      Kernel.p options
       sql = "SELECT * INTO OUTFILE '#{options[:filepath]}' " 
       
       sql += " FIELDS TERMINATED BY '\\0' ESCAPED BY '' OPTIONALLY ENCLOSED BY '\\\"'  LINES TERMINATED BY '\\n'"
@@ -143,13 +145,17 @@ module Myreplicator
     # Location of the output file needs to have 777 perms
     ##
     def self.mysql_export_outfile *args
+      Kernel.p "===== mysql_export_outfile OPTIONS ====="
+       
       options = args.extract_options!
+      Kernel.p options
       options.reverse_merge! :flags => []
-      db = options[:db]
+      db = options[:source_schema]
 
       # Database host when ssh'ed into the db server
       db_host = "127.0.0.1"
-      
+      Kernel.p "===== mysql_export_outfile ssh_configs ====="
+      Kernel.p ssh_configs(db)
       if !ssh_configs(db)["ssh_db_host"].blank?
         db_host =  ssh_configs(db)["ssh_db_host"]
       elsif !db_configs(db)["host"].blank?

data/lib/loader/loader.rb

@@ -31,7 +31,7 @@ module Myreplicator
     def self.load
       initials = []
       incrementals = []
-      all_files = Loader.metadata_files
+      all_files = Myreplicator::Loader.metadata_files
 
       all_files.each do |m|
         if m.export_type == "initial"
@@ -80,7 +80,11 @@ module Myreplicator
                   :export_id => metadata.export_id) do |log|
 
             if Loader.transfer_completed? metadata
-              Loader.initial_load metadata
+              if metadata.export_to == "vertica"
+                Loader.incremental_load metadata
+              else
+                Loader.initial_load metadata
+              end
               Loader.cleanup metadata
             end
 
@@ -107,7 +111,7 @@ module Myreplicator
                     :file => metadata.filename, 
                     :export_id => metadata.export_id) do |log|
     
-              if Loader.transfer_completed? metadata
+              if Loader.transfer_completed? metadata            
                 Loader.incremental_load metadata
                 Loader.cleanup metadata
               end
@@ -152,6 +156,7 @@ module Myreplicator
     ##
     def self.initial_load metadata
       exp = Export.find(metadata.export_id)
+      Kernel.p "===== unzip ====="
       Loader.unzip(metadata.filename)
       metadata.zipped = false
       
@@ -176,22 +181,16 @@ module Myreplicator
       Loader.unzip(metadata.filename)
       metadata.zipped = false
       
-      options = {:table_name => exp.table_name, :db => exp.destination_schema,
-        :filepath => metadata.destination_filepath(tmp_dir), :source_schema => exp.source_schema}
-      
-      if metadata.export_type == "incremental_outfile"
-        options[:fields_terminated_by] = "\\0"
-        options[:lines_terminated_by] = "\\n"
-      end
+      options = {:table_name => exp.table_name, 
+        :db => exp.destination_schema,
+        :filepath => metadata.destination_filepath(tmp_dir), 
+        :source_schema => exp.source_schema,      
+        :fields_terminated_by => "\\0",
+        :lines_terminated_by => "\\n"}
       
       case metadata.export_to 
       when "vertica"
-        options = {:table_name => exp.table_name, :db => ActiveRecord::Base.configurations["vertica"]["database"],
-          :filepath => metadata.destination_filepath(tmp_dir), :source_schema => exp.source_schema, :export_id => metadata.export_id}
-        options[:destination_schema] = exp.destination_schema
-        Kernel.p "===== LOAD TO VERTICA ====="
-        Kernel.p options
-        Myreplicator::VerticaLoader.load options
+        Loader.load_to_vertica options, metadata, exp
       when "mysql"
         cmd = ImportSql.load_data_infile(options)
         puts cmd
@@ -204,6 +203,25 @@ module Myreplicator
       end #case  
     end
 
+    ##
+    # Load to Vertica
+    ##
+    def self.load_to_vertica options, metadata, exp
+      options = {:table_name => exp.table_name, 
+        :db => ActiveRecord::Base.configurations["vertica"]["database"],
+        :filepath => metadata.destination_filepath(tmp_dir), 
+        :source_schema => exp.source_schema, :export_id => metadata.export_id,
+        :metadata => metadata
+      }
+      
+      options[:destination_schema] = exp.destination_schema
+      
+      result = Myreplicator::VerticaLoader.load options
+      
+      ##TO DO: Handle unsuccessful vertica loads here
+
+    end
+
     ##
     # Returns true if the transfer of the file
     # being loaded is completed
@@ -222,7 +240,7 @@ module Myreplicator
     ##
     def self.cleanup metadata
       puts "Cleaning up..."
-      FileUtils.rm "#{metadata.destination_filepath(tmp_dir)}.json" # json file
+      FileUtils.rm metadata.metadata_filepath(tmp_dir) # json file
       FileUtils.rm metadata.destination_filepath(tmp_dir) # dump file
     end
 
@@ -259,8 +277,27 @@ module Myreplicator
       return files
     end
 
+    ##
+    # Clears files that are older than the passed metadata file.
+    # Note: This methoded is provided to ensure no old incremental files
+    # ever get loaded after the schema change algorithm has been applied 
+    ##
+    def self.clear_older_files metadata
+      files = Loader.metadata_files
+      Kernel.p "===== clear old files ====="
+      Kernel.p metadata
+      Kernel.p files
+      max_date = DateTime.strptime metadata.export_time
+      files.each do |m|
+        if metadata.export_id == m.export_id
+          if max_date > DateTime.strptime(m.export_time)
+            Loader.cleanup m if metadata.filepath != m.filepath
+          end 
+        end
+      end     
+    end
+
     def self.mysql_table_definition options
-      Kernel.p options
       sql = "SELECT table_schema, table_name, column_name, is_nullable, data_type, column_type, column_key "
       sql += "FROM INFORMATION_SCHEMA.COLUMNS where table_name = '#{options[:table]}' "
       sql += "and table_schema = '#{options[:source_schema]}';"

data/lib/loader/vertica/vertica_loader.rb

@@ -4,6 +4,8 @@ module Myreplicator
 
       def create_table *args
         options = args.extract_options!
+        Kernel.p "===== OPTION ====="
+              puts options
         columns = []
         options[:mysql_schema].each(:as => :hash) do |row|
           columns << row
@@ -32,124 +34,124 @@ module Myreplicator
       ##
       # rasing a concern: using the same schema or the tmp schema for the tmp table? Vertica doesn't lock the schema
       def apply_schema_change options, temp_table
-        Myreplicator::VerticaLoader.create_table({:mysql_schema => options[:mysql_schema],
-                                                            :vertica_db => options[:vertica_db], 
-                                                            :vertica_schema => options[:vertica_schema],
-                                                            :table => temp_table,
-                                                            :mysql_table => options[:table]
-                                                          })
-      end
+        Kernel.p "+++++++++++++++++ options "
+        puts options
+        VerticaLoader.create_table({:mysql_schema => options[:mysql_schema],
+                                     :vertica_db => options[:vertica_db], 
+                                     :vertica_schema => options[:vertica_schema],
+                                     :table => temp_table,
+                                     :mysql_table => options[:table]})
 
-      def full_load options, temp_table
         export_id = options[:export_id]
         new_options = prepare_options options
-        begin
-          a = Myreplicator::Export.find(export_id)
-          a.max_incremental_value = "0"
-          a.save!
-          a.export
-          b = Myreplicator::Transporter
-          b.perform
-          file = File.join(Myreplicator.app_root,"tmp", "myreplicator", "#{a.filename}")
-          `gunzip #{file}.gz`
-          new_options[:file] = file
-          new_options[:table] = temp_table
-          new_options[:schema] = options[:vertica_schema]
-          
-          vertica_copy new_options
-          
-          #drop
-          FileUtils.rm file
-          FileUtils.rm "#{file}.json"
-          sql = "DROP TABLE #{options[:vertica_db]}.#{options[:vertica_schema]}.#{options[:table]} CASCADE;"
-          VerticaDb::Base.connection.execute sql
-          #rename
-          sql = "ALTER TABLE #{options[:vertica_db]}.#{options[:vertica_schema]}.#{temp_table} RENAME TO #{options[:table]};"
-          VerticaDb::Base.connection.execute sql
-        rescue Exception => e
-          raise e.message 
-        ensure
-          
-        end
+        new_options[:file] = options[:filepath]
+        new_options[:table] = temp_table
+        new_options[:schema] = options[:vertica_schema]
+        
+         
+        vertica_copy new_options
+        Kernel.p "+++++++++++++++++ new_options "
+        puts new_options
+        puts options
+        
+        sql = "DROP TABLE IF EXISTS #{options[:vertica_db]}.#{options[:vertica_schema]}.#{options[:table]} CASCADE;"
+        VerticaDb::Base.connection.execute sql
+        #rename
+        sql = "ALTER TABLE #{options[:vertica_db]}.#{options[:vertica_schema]}.#{temp_table} RENAME TO #{options[:table]};"
+        VerticaDb::Base.connection.execute sql
+
       end
+      
+      def create_temp_table *args
+        options = args.extract_options!
+        temp_table_name = "temp_" + options[:table] + DateTime.now.strftime('%Y%m%d_%H%M%S').to_s
 
-      # def create_all_tables db
-      #   tables = Loader::SourceDb.get_tables(db)
-      #   sqls = {}
-      #   tables.each do |table|
-      #     puts "Creating #{db}.#{table}"
-      #     sql = "DROP TABLE IF EXISTS #{db}.#{table} CASCADE;"
-      #     VerticaDb::Base.connection.execute sql
-      #     sql = Loader::VerticaLoader.create_table(:vertica_db => "bidw",
-      #     :vertica_table => table,
-      #     :vertica_schema => db,
-      #     :table => table,
-      #     :db => db)
-      #     sqls["#{table}"] = sql
-      #     VerticaDb::Base.connection.execute sql
-      #   end
-      # end
-    
+        VerticaLoader.create_table({:mysql_schema => options[:mysql_schema],
+                                     :vertica_db => options[:vertica_db], 
+                                     :vertica_schema => options[:vertica_schema],
+                                     :table => temp_table_name,
+                                     :mysql_table => options[:table]})
+        return temp_table_name
+      end
+   
       def prepare_options *args
-        options = args.extract_options!
-        vertica_options =ActiveRecord::Base.configurations["vertica"]
-        
-        result = options.clone
-        result.reverse_merge!(
-          :host => vertica_options["host"],
-          :user => vertica_options["username"],
-          :pass => vertica_options["password"],
-          :db   => vertica_options["database"],
-          :schema => options[:destination_schema],
-          :table => options[:table_name],
-          :file => options[:filepath],
-          :delimiter => "\\0",
-          :null_value => "NULL",
-          :enclosed => ""
-        )
+        options = args.extract_options!.clone
+        Kernel.p "===== OPTION  [options[:db]] ====="
+        puts options
+        # How not to hard code the vertica connection config ?
+        vertica_options = ActiveRecord::Base.configurations["vertica"]
+
+        options.reverse_merge!(:host => vertica_options["host"],
+                              :user => vertica_options["username"],
+                              :pass => vertica_options["password"],
+                              :db   => vertica_options["database"],
+                              :schema => options[:destination_schema],
+                              :table => options[:table_name],
+                              :file => options[:filepath],
+                              :delimiter => "\\0",
+                              :null_value => "NULL",
+                              :enclosed => "")
+      # working now but should fix this 
         if !vertica_options["vsql"].blank?
-          result.reverse_merge!(
-            :vsql => vertica_options["vsql"]
-            )
+          options.reverse_merge!(:vsql => vertica_options["vsql"])
         else
-          result.reverse_merge!(
-            :vsql => "/opt/vertica/bin/vsql"
-          )
+          options.reverse_merge!(:vsql => "/opt/vertica/bin/vsql")
         end
         
-        return result  
+        return options  
       end
       
       # Loader::VerticaLoader.load({:schema => "king", :table => "category_overview_data", :file => "tmp/vertica/category_overview_data.tsv", :null_value => "NULL"})
+      # check for export_type!
       def load *args
         options = args.extract_options!
-        #options = {:table => "app_csvs", :db => "public", :source_schema => "okl_dev"}
+        metadata = options[:metadata]
+        Kernel.p "===== metadata ====="
+        Kernel.p metadata
+        Kernel.p options
+        #options = {:table => "app_csvs", :destination_schema => "public", :source_schema => "okl_dev"}
+        #options = {:table => "actucast_appeal", :destination_schema => "public", :source_schema => "raw_sources"}
         schema_check = Myreplicator::MysqlExporter.schema_changed?(:table => options[:table_name], 
                                                      :destination_schema => options[:destination_schema], 
                                                      :source_schema => options[:source_schema])
-        
+        Kernel.p "===== schema_check ====="
+        Kernel.p schema_check
+        Kernel.p schema_check[:mysql_schema]
         #create a temp table
         temp_table = "temp_" + options[:table_name] + DateTime.now.strftime('%Y%m%d_%H%M%S').to_s
         ops = {:mysql_schema => schema_check[:mysql_schema],
-               :vertica_db => options[:db],
-               :vertica_schema => options[:destination_schema],
-               :table => options[:table_name],
-               :export_id => options[:export_id]
+          :vertica_db => options[:db],
+          :vertica_schema => options[:destination_schema],
+          :table => options[:table_name],
+          :export_id => options[:export_id],
+          :filepath => options[:filepath]
         }
-        begin
-          if schema_check[:new]
-            create_table(ops)
+        Kernel.p "===== schema_check[:mysql_schema] ====="
+        Kernel.p ops
+        if schema_check[:new]
+          create_table(ops)
+          #LOAD DATA IN
+          vertica_copy options
+        elsif schema_check[:changed]
+          if metadata.export_type == 'initial'
+            Loader.clear_older_files metadata  # clear old incremental files
             apply_schema_change(ops, temp_table)
-            #vertica_copy options
-            full_load(ops, temp_table) 
-          elsif schema_check[:changed]
-            apply_schema_change(ops, temp_table)
-            full_load(ops, temp_table)
           else
-            vertica_copy options
+            Loader.cleanup metadata #Remove incremental file
           end
-        rescue Exception => e
-          raise e.message
+        else
+          temp_table = create_temp_table ops
+          options[:table] = temp_table
+          Kernel.p "===== COPY TO TEMP TABLE #{temp_table} ====="
+          vertica_copy options
+          options.reverse_merge!(:temp_table => "#{temp_table}")
+          options[:table] = options[:table_name]
+          Kernel.p "===== MERGE ====="
+          vertica_merge options
+          #drop the temp table
+          Kernel.p "===== DROP TEMP TABLE ====="
+          sql = "DROP TABLE IF EXISTS #{options[:db]}.#{options[:destination_schema]}.#{temp_table} CASCADE;"
+          VerticaDb::Base.connection.execute sql
         end
       end
       
@@ -161,17 +163,16 @@ module Myreplicator
           raise "No input file"
         end
         
-        begin
-          process_file(:file => prepared_options[:file], :list_of_nulls => list_of_nulls, :null_value => prepared_options[:null_value])
-          cmd = get_vsql_command(prepared_options)
-          Kernel.p cmd
-          system(cmd)
-        rescue Exception => e
-          raise e.message
-        end
+        process_file(:file => prepared_options[:file], 
+                     :list_of_nulls => list_of_nulls,
+                     :null_value => prepared_options[:null_value])
+
+        cmd = get_vsql_copy_command(prepared_options)
+        puts cmd
+        system(cmd)
       end
         
-      def get_vsql_command prepared_options
+      def get_vsql_copy_command prepared_options
         file_extension = prepared_options[:file].split('.').last
         file_handler = ""
         file_handler = "GZIP" if file_extension == "gz" 
@@ -229,6 +230,145 @@ module Myreplicator
         system(cmd2)
       end
 
+      def get_mysql_keys mysql_schema_simple_form
+        result = []
+        mysql_schema_simple_form.each do |col|
+          if col["column_key"] == "PRI"
+            result << col["column_name"]
+          end
+        end
+        return result
+      end
+      
+      def get_mysql_none_keys mysql_schema_simple_form
+        result = []
+        mysql_schema_simple_form.each do |col|
+          if col["column_key"].blank?
+            result << col["column_name"]
+          end
+        end
+        return result
+      end
+      
+      def get_mysql_inserted_columns mysql_schema_simple_form
+        result = []
+        mysql_schema_simple_form.each do |col|
+          result << col["column_name"]
+        end
+        return result
+      end
+      
+      def get_vsql_merge_command options, keys, none_keys, inserted_columns
+        Kernel.p "===== Merge Options ====="
+        Kernel.p options
+        a = prepare_options options
+        Kernel.p a
+        prepared_options = options
+        sql = "MERGE INTO "
+        sql+= "#{prepared_options[:db]}.#{prepared_options[:schema]}.#{prepared_options[:table]} target "
+        sql+= "USING #{prepared_options[:db]}.#{prepared_options[:schema]}.#{prepared_options[:temp_table]} source "
+        sql+= "ON "
+        count = 0
+        keys.each do |k|
+          if count < 1 
+            sql += "source.#{k} = target.#{k} "
+          else
+            sql += "AND source.#{k} = target.#{k} "
+          end
+          count += 1
+        end
+        sql+= "WHEN MATCHED THEN "
+        sql+= "UPDATE SET "
+        count = 1
+        none_keys.each do |nk|
+          if count < none_keys.size
+            sql+= "#{nk} = source.#{nk}, "
+          else
+            sql+= "#{nk} = source.#{nk} "
+          end
+          count += 1
+        end
+        sql+= "WHEN NOT MATCHED THEN "
+        sql+= "INSERT "
+        #count = 1
+        #inserted_columns.each do |col|
+        #  if count < inserted_columns.size
+        #    sql+= "#{col}, "
+        #  else
+        #    sql+= "#{col} "
+        #  end
+        #  count += 1
+        #end
+        count = 1
+        sql+= " VALUES ("
+        inserted_columns.each do |col|
+          if count < inserted_columns.size
+            sql+= "source.#{col}, "
+          else
+            sql+= "source.#{col}) "
+          end
+          count += 1
+        end  
+        sql+= ";"  
+        cmd = "#{prepared_options[:vsql]} -h #{prepared_options[:host]} -U #{prepared_options[:user]} -w #{prepared_options[:pass]} -d #{prepared_options[:db]} -c \"#{sql}\""
+        return cmd    
+      end
+        
+      def vertica_merge *args
+        options = args.extract_options!
+        metadata = options[:metadata]
+        Kernel.p "===== MERGE metadata ====="
+        Kernel.p metadata
+        ops = {:table => options[:table_name], 
+        :destination_schema => options[:destination_schema], 
+        :source_schema => options[:source_schema]}
+        mysql_schema = Loader.mysql_table_definition(options)
+        vertica_schema = VerticaLoader.destination_table_vertica(options)
+        mysql_schema_simple_form = MysqlExporter.get_mysql_schema_rows mysql_schema
+        # get the column(s) that is(are) used as the primary key
+        keys = get_mysql_keys mysql_schema_simple_form
+        # get the non key coluns 
+        none_keys = get_mysql_none_keys mysql_schema_simple_form
+        # get the column to put in the insert part
+        inserted_columns = get_mysql_inserted_columns mysql_schema_simple_form
+        #get the vsql merge command 
+        cmd = get_vsql_merge_command options, keys, none_keys, inserted_columns 
+        #execute    
+        puts cmd
+        begin
+          result = `#{cmd} 2>&1`
+          if result[0..4] == "ERROR"
+            Loader.cleanup metadata
+            sql = "DROP TABLE IF EXISTS #{options[:db]}.#{options[:destination_schema]}.#{options[:temp_table]} CASCADE;"
+            Kernel.p "===== DROP CMD ====="
+            Kernel.p sql
+            VerticaDb::Base.connection.execute sql
+            raise result
+          end
+        rescue Exception => e
+          raise e.message 
+        ensure
+          # place holder
+        end
+      end
+
+      # def create_all_tables db
+      #   tables = Loader::SourceDb.get_tables(db)
+      #   sqls = {}
+      #   tables.each do |table|
+      #     puts "Creating #{db}.#{table}"
+      #     sql = "DROP TABLE IF EXISTS #{db}.#{table} CASCADE;"
+      #     VerticaDb::Base.connection.execute sql
+      #     sql = Loader::VerticaLoader.create_table(:vertica_db => "bidw",
+      #     :vertica_table => table,
+      #     :vertica_schema => db,
+      #     :table => table,
+      #     :db => db)
+      #     sqls["#{table}"] = sql
+      #     VerticaDb::Base.connection.execute sql
+      #   end
+      # end
+
     end
   end
 end

data/lib/myreplicator/version.rb

@@ -1,3 +1,3 @@
 module Myreplicator
-  VERSION = "1.1.7"
+  VERSION = "1.1.9"
 end

data/test/dummy/log/development.log

@@ -4086,3 +4086,275 @@ Connecting to database specified by database.yml
 Connecting to database specified by database.yml
 Connecting to database specified by database.yml
 Connecting to database specified by database.yml
+Connecting to database specified by database.yml
+
+
+Started GET "/myreplicator/exports" for 127.0.0.1 at 2013-03-08 14:24:06 -0800
+Processing by Myreplicator::ExportsController#index as HTML
+   (0.3ms)  SELECT COUNT(*) FROM `myreplicator_exports` 
+  Myreplicator::Export Load (0.5ms)  SELECT `myreplicator_exports`.* FROM `myreplicator_exports` ORDER BY source_schema asc LIMIT 30 OFFSET 0
+  Rendered /home/sasan/workspace/myreplicator/app/views/myreplicator/exports/index.html.erb within layouts/myreplicator/application (73.0ms)
+Completed 200 OK in 361ms (Views: 356.6ms | ActiveRecord: 2.0ms)
+
+
+Started GET "/assets/myreplicator/application.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/application.css - 304 Not Modified (32ms)
+
+
+Started GET "/assets/myreplicator/chosen.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/chosen.css - 304 Not Modified (3ms)
+
+
+Started GET "/assets/myreplicator/exports.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/exports.css - 304 Not Modified (2ms)
+
+
+Started GET "/assets/jquery.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /jquery.js - 304 Not Modified (3ms)
+
+
+Started GET "/assets/myreplicator/tipTip.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/tipTip.css - 304 Not Modified (1ms)
+
+
+Started GET "/assets/myreplicator/select2.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/select2.css - 304 Not Modified (2ms)
+
+
+Started GET "/assets/jquery-ui.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /jquery-ui.js - 304 Not Modified (3ms)
+
+
+Started GET "/assets/jquery_ujs.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /jquery_ujs.js - 304 Not Modified (2ms)
+
+
+Started GET "/assets/myreplicator/chosen.jquery.min.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/chosen.jquery.min.js - 304 Not Modified (1ms)
+
+
+Started GET "/assets/myreplicator/cronwtf.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/cronwtf.js - 304 Not Modified (1ms)
+
+
+Started GET "/assets/myreplicator/exports.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/exports.js - 304 Not Modified (1ms)
+
+
+Started GET "/assets/myreplicator/jquery.tipTip.minified.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/jquery.tipTip.minified.js - 304 Not Modified (1ms)
+
+
+Started GET "/assets/myreplicator/select2.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/select2.js - 304 Not Modified (29ms)
+
+
+Started GET "/assets/myreplicator/application.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/application.js - 304 Not Modified (8ms)
+
+
+Started GET "/assets/myreplicator/plus.png" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/plus.png - 304 Not Modified (31ms)
+
+
+Started GET "/assets/myreplicator/desc-white.gif" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/desc-white.gif - 304 Not Modified (7ms)
+
+
+Started GET "/assets/myreplicator/bg.gif" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/bg.gif - 304 Not Modified (56ms)
+
+
+Started GET "/assets/myreplicator/clipboard-list.png" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/clipboard-list.png - 304 Not Modified (3ms)
+
+
+Started GET "/assets/myreplicator/gear.png" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/gear.png - 304 Not Modified (7ms)
+
+
+Started GET "/assets/myreplicator/status.png" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/status.png - 304 Not Modified (10ms)
+
+
+Started GET "/assets/myreplicator/FrancoisOne.ttf" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/FrancoisOne.ttf - 304 Not Modified (13ms)
+
+
+Started GET "/assets/myreplicator/websymbols-regular-webfont.woff" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/websymbols-regular-webfont.woff - 304 Not Modified (55ms)
+
+
+Started GET "/assets/myreplicator/cross.png" for 127.0.0.1 at 2013-03-08 14:24:07 -0800
+Served asset /myreplicator/cross.png - 304 Not Modified (11ms)
+
+
+Started GET "/myreplicator/exports/new" for 127.0.0.1 at 2013-03-08 14:24:09 -0800
+Processing by Myreplicator::ExportsController#new as HTML
+  Rendered /home/sasan/workspace/myreplicator/app/views/myreplicator/exports/_form.html.erb (68.7ms)
+  Rendered /home/sasan/workspace/myreplicator/app/views/myreplicator/exports/new.html.erb within layouts/myreplicator/application (76.6ms)
+Completed 200 OK in 553ms (Views: 102.5ms | ActiveRecord: 76.8ms)
+
+
+Started GET "/assets/myreplicator/application.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/application.css - 304 Not Modified (4ms)
+
+
+Started GET "/assets/myreplicator/exports.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/exports.css - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/chosen.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/chosen.css - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/select2.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/select2.css - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/tipTip.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/tipTip.css - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/cronwtf.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/cronwtf.js - 304 Not Modified (0ms)
+
+
+Started GET "/assets/jquery-ui.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /jquery-ui.js - 304 Not Modified (2ms)
+
+
+Started GET "/assets/jquery_ujs.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /jquery_ujs.js - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/chosen.jquery.min.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/chosen.jquery.min.js - 304 Not Modified (0ms)
+
+
+Started GET "/assets/jquery.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /jquery.js - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/exports.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/exports.js - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/jquery.tipTip.minified.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/jquery.tipTip.minified.js - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/select2.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/select2.js - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/application.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/application.js - 304 Not Modified (2ms)
+
+
+Started GET "/assets/myreplicator/cross.png" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/cross.png - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/FrancoisOne.ttf" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/FrancoisOne.ttf - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/websymbols-regular-webfont.woff" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/websymbols-regular-webfont.woff - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/select2.png" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/select2.png - 304 Not Modified (21ms)
+
+
+Started GET "/assets/myreplicator/slider_handle.png" for 127.0.0.1 at 2013-03-08 14:24:10 -0800
+Served asset /myreplicator/slider_handle.png - 304 Not Modified (11ms)
+
+
+Started POST "/myreplicator/exports" for 127.0.0.1 at 2013-03-08 14:24:33 -0800
+Processing by Myreplicator::ExportsController#create as HTML
+  Parameters: {"utf8"=>"✓", "authenticity_token"=>"QmN8JvOm6ve0DKeySyGW8m4j4mX7L4aFx3WoUG5Iguk=", "export"=>{"export_to"=>"vertica", "export_type"=>"incremental", "source_schema"=>"okl_dev", "destination_schema"=>"public", "table_name"=>"activation_cohorts", "incremental_column"=>"modified_date", "incremental_column_type"=>"", "max_incremental_value"=>"", "s3_path"=>"", "active"=>"true", "cron"=>"*/23 * * * *"}, "commit"=>"Create Export"}
+   (0.1ms)  BEGIN
+  SQL (0.4ms)  INSERT INTO `myreplicator_exports` (`active`, `created_at`, `cron`, `destination_schema`, `error`, `export_finished_at`, `export_started_at`, `export_to`, `export_type`, `exporter_pid`, `incremental_column`, `incremental_column_type`, `max_incremental_value`, `s3_path`, `source_schema`, `state`, `table_name`, `updated_at`) VALUES (1, '2013-03-08 22:24:34', '*/23 * * * *', 'public', NULL, NULL, NULL, 'vertica', 'incremental', NULL, 'modified_date', '', '', '', 'okl_dev', 'new', 'activation_cohorts', '2013-03-08 22:24:34')
+   (42.3ms)  COMMIT
+  Myreplicator::Export Load (0.3ms)  SELECT `myreplicator_exports`.* FROM `myreplicator_exports` 
+Redirected to http://localhost:3000/myreplicator/exports/4
+Completed 302 Found in 561ms (ActiveRecord: 120.4ms)
+
+
+Started GET "/myreplicator/exports/4" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Processing by Myreplicator::ExportsController#show as HTML
+  Parameters: {"id"=>"4"}
+  Myreplicator::Export Load (0.3ms)  SELECT `myreplicator_exports`.* FROM `myreplicator_exports` WHERE `myreplicator_exports`.`id` = 4 LIMIT 1
+  Rendered /home/sasan/workspace/myreplicator/app/views/myreplicator/exports/show.html.erb within layouts/myreplicator/application (1.5ms)
+Completed 200 OK in 23ms (Views: 20.7ms | ActiveRecord: 0.3ms)
+
+
+Started GET "/assets/myreplicator/application.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/application.css - 304 Not Modified (5ms)
+
+
+Started GET "/assets/myreplicator/tipTip.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/tipTip.css - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/chosen.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/chosen.css - 304 Not Modified (1ms)
+
+
+Started GET "/assets/myreplicator/exports.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/exports.css - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/select2.css?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/select2.css - 304 Not Modified (0ms)
+
+
+Started GET "/assets/jquery.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /jquery.js - 304 Not Modified (1ms)
+
+
+Started GET "/assets/jquery-ui.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /jquery-ui.js - 304 Not Modified (1ms)
+
+
+Started GET "/assets/jquery_ujs.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /jquery_ujs.js - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/cronwtf.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/cronwtf.js - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/chosen.jquery.min.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/chosen.jquery.min.js - 304 Not Modified (2ms)
+
+
+Started GET "/assets/myreplicator/exports.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/exports.js - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/jquery.tipTip.minified.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/jquery.tipTip.minified.js - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/select2.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/select2.js - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/application.js?body=1" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/application.js - 304 Not Modified (2ms)
+
+
+Started GET "/assets/myreplicator/FrancoisOne.ttf" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/FrancoisOne.ttf - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/websymbols-regular-webfont.woff" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/websymbols-regular-webfont.woff - 304 Not Modified (0ms)
+
+
+Started GET "/assets/myreplicator/gear.png" for 127.0.0.1 at 2013-03-08 14:24:34 -0800
+Served asset /myreplicator/gear.png - 304 Not Modified (0ms)
+Connecting to database specified by database.yml

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: myreplicator
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.1.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-07 00:00:00.000000000 Z
+date: 2013-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -370,7 +370,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 45298977780065472
+      hash: -3137880076696344108
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -379,7 +379,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 45298977780065472
+      hash: -3137880076696344108
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23