myreplicator 1.0.6 → 1.1.0

data/app/views/myreplicator/exports/_form.html.erb

@@ -1,6 +1,6 @@
 <% 
 #rather than yaml arrays of the drop downs
-export_to = ["destination_db","s3","both"]
+export_to = ["destination_db","backup","vertica"]
 export_type = ["incremental","fulldump"]
 %>
 <%= form_for(@export) do |f| %>
@@ -16,6 +16,18 @@ export_type = ["incremental","fulldump"]
     </div>
   <% end %>
   <div class="form-section">
+<label>Export to</label>
+<select name="export[export_to]" id="export_to" class="chosen" data-placeholder="Select export destination">
+  <% export_to.each do |val| %>
+  <option value="<%= val %>" <% if @export.export_to == val %>SELECTED<% end %>><%= val %></option>
+  <% end %> 
+</select>
+<label>Export type</label>
+<select name="export[export_type]" id="export_type" class="chosen" data-placeholder="Select your export type">
+  <% export_type.each do |val| %>
+  <option value="<%= val %>" <% if @export.export_type == val %>SELECTED<% end %>><%= val %></option>
+  <% end %>
+</select>    
 <label>Source Schema</label>
 <%= f.select :source_schema, @dbs %>
 <label>Destination Schema</label>
@@ -28,20 +40,8 @@ export_type = ["incremental","fulldump"]
 <%= f.text_field :incremental_column_type %>
 <label>Maximum Incremental Value</label>
 <%= f.text_field :max_incremental_value %>
-<label>S3 Path</label>
+<label>Store a copy in</label>
 <%= f.text_field :s3_path %>
-<label>Export to</label>
-<select name="export[export_to]" id="export_to" class="chosen" data-placeholder="Select export destination">
-  <% export_to.each do |val| %>
-  <option value="<%= val %>" <% if @export.export_to == val %>SELECTED<% end %>><%= val %></option>
-  <% end %> 
-</select>
-<label>Export type</label>
-<select name="export[export_type]" id="export_type" class="chosen" data-placeholder="Select your export type">
-  <% export_type.each do |val| %>
-  <option value="<%= val %>" <% if @export.export_type == val %>SELECTED<% end %>><%= val %></option>
-  <% end %>
-</select>
 <label>Active</label>
 <select name="export[active]" id="active" class="chosen">
   <option value="true" <% if @export.active == true %>SELECTED<% end %>>true</option>
@@ -124,8 +124,7 @@ export_type = ["incremental","fulldump"]
     <%= link_to content_tag(:span, 'cancel'), exports_path, :class => "btn left action cancel" %>
   </div>
 <% end %>
-<%= javascript_include_tag "myreplicator/chosen.jquery.min" %>
-<%= javascript_include_tag "myreplicator/cronwtf.min" %>
+
 <script>
 var dbs = {
 <% @tables.each do |key,values| %>
@@ -138,13 +137,28 @@ var dbs = {
 }
 $(function(){
   CronUI.translate();
-  $(".chosen").chosen();
-  $("#export_destination_schema,#export_table_name").chosen();
+  $(".chosen").select2();
+  var $destination = $("#export_destination_schema").clone();
+  var $table_name = $("#export_table_name").clone()
+  $("#export_destination_schema").select2();
+  $("#export_table_name").select2();
   $(".cron").chosen().change(function(){CronUI.translate()});
-  $("#export_source_schema").chosen().change(function(){
+  $("#export_source_schema").select2().change(function(){
     exportSchemaSelect($(this).val())
   });
   exportSchemaSelect($("#export_source_schema").val());
+
+  $("#export_to").change(function(){
+    var exp = $(this).val();
+    if(exp == 'vertica'){
+      $("#export_destination_schema").select2("destroy").before('<input type="text" id="export_destination_schema" name="export[destination_schema]"/>').remove();
+    }else{
+      if($("#s2id_export_destination_schema").length < 1){
+        $("#export_destination_schema").before($destination).remove();
+        $("#export_destination_schema").select2();
+      }
+    }
+  })
   $("#cron-min").slider({
     min: 1, 
     max: 59,
@@ -218,18 +232,25 @@ function editInit(){
     }
   }
   CronUI.translate();
-  $("#export_table_name").val("<%= @export.table_name %>").trigger("liszt:updated");
+  <% if @export.export_to != 'vertica' %>
+  $("#export_table_name").select2("val","<%= @export.table_name %>")
+  <% else %>
+    $("#export_destination_schema").select2("destroy").before('<input type="text" id="export_destination_schema" name="export[destination_schema]" value="<%= @export.destination_schema %>"/>').remove();
+  <% end %>
 }
 
 function exportSchemaSelect(val){
     var target = $("#export_table_name");
-    var tables = dbs[val];
-    var l = tables.length;
-    var options = "";
-    for(i=0;i<l;i++){
-      options += "<option value='"+tables[i]+"'>"+tables[i]+"</option>";
+    if(target.hasClass("select2-offscreen")){
+      target.select2("destroy")
+      var tables = dbs[val];
+      var l = tables.length;
+      var options = "";
+      for(i=0;i<l;i++){
+        options += "<option value='"+tables[i]+"'>"+tables[i]+"</option>";
+      }
+      target.val('').find("option").remove()
+      target.append(options).select2();
     }
-    target.val('').find("option").remove().trigger("liszt:updated");
-    target.append(options).trigger("liszt:updated");
 }
 </script>

data/app/views/myreplicator/exports/index.html.erb

@@ -8,14 +8,14 @@
   <table class="data-grid">
     <thead>
       <tr>
+       <th><%= sortable "export_type" %></th> 
+       <th><%= sortable "export_to", "Export Desitination" %></th>       
        <th colspan="2"><%= sortable "source_schema" %></th>
        <th><%= sortable "destination_schema" %></th>
        <th><%= sortable "table_name" %></th>
        <th><%= sortable "incremental_column" %></th>
        <th><%= sortable "incremental_column_type" %></th>
        <th><%= sortable "max_incremental_value" %></th>
-       <th><%= sortable "export_to", "Export Desitination" %></th>
-       <th><%= sortable "export_type" %></th> 
        <th><%= sortable "s3_path" %></th>
        <th><%= sortable "cron" %></th> 
        <th class="center">Actions</th>
@@ -24,6 +24,8 @@
     <tbody>
     <% @exports.each do |export| %>
       <tr>
+        <td><%= export.export_type %></td> 
+        <td><%= export.export_to %></td>        
         <td class="state"><span class="status <% if export.active %>active<% else %>inactive<% end %>" title="<% if export.active %>Active<% else %>Inactive<% end %>"></span></td> 
         <td class="source"><%= export.source_schema %></td>
         <td><%= export.destination_schema %></td>
@@ -31,8 +33,6 @@
         <td><%= export.incremental_column %></td>
         <td><%= export.incremental_column_type %></td>
         <td><%= export.max_incremental_value %></td>
-        <td><%= export.export_to %></td>
-        <td><%= export.export_type %></td> 
         <td><%= export.s3_path %></td>
         <td><span class="cron" data-cron="<%= export.cron %>" title=""><%= export.cron %></span></td>     
         <td>

data/db/migrate/20130213211927_create_myreplicator_vertica_exports.rb

@@ -0,0 +1,22 @@
+class CreateMyreplicatorVerticaExports < ActiveRecord::Migration
+  def change
+    create_table :myreplicator_vertica_exports do |t|
+      t.string :database
+      t.string :schema
+      t.string :table_name
+      t.string :file_path
+      t.string :file_format
+      t.string :delimiter
+      t.string :field_enclosed_by
+      t.string :line_terminated_by
+      t.string :cron
+      t.string :state, :default => "new"
+      t.text :error
+      t.boolean :active, :default => true
+      t.integer :exporter_pid
+      t.datetime :export_started_at, :default => nil
+      t.datetime :export_finished_at, :default => nil
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20130213211927_create_myreplicator_vertica_exports.rb~

@@ -0,0 +1,8 @@
+class CreateMyreplicatorVerticaExports < ActiveRecord::Migration
+  def change
+    create_table :myreplicator_vertica_exports do |t|
+
+      t.timestamps
+    end
+  end
+end

data/lib/exporter/export_metadata.rb

@@ -5,6 +5,7 @@ module Myreplicator
 
     attr_accessor(:export_time, 
                   :export_finished_at, 
+                  :export_to,
                   :table, 
                   :database, 
                   :state, 
@@ -13,6 +14,7 @@ module Myreplicator
                   :incremental_val,
                   :ssh,
                   :export_type,
+                  :store_in,
                   :on_duplicate,
                   :filepath,
                   :zipped,
@@ -158,7 +160,9 @@ module Myreplicator
         :export_id => @export_id,
         :filepath => @filepath,
         :zipped => @zipped,
-        :export_type => @export_type
+        :export_type => @export_type,
+        :export_to => @export_to,
+        :store_in => @store_in
       }
       return obj.to_json
     end
@@ -203,6 +207,8 @@ module Myreplicator
       @on_duplicate = options[:on_duplicate] if options[:on_duplicate]
       @export_type = options[:export_type] if options[:export_type]
       @zipped = options[:zipped].nil? ? false : options[:zipped]
+      @store_in = options[:store_in] if options[:store_in]
+      @export_to = options[:export_to] if options[:export_to]
       @ssh = nil
 
       @success_callbacks = []

data/lib/exporter/mysql_exporter.rb

@@ -15,19 +15,19 @@ module Myreplicator
 
       ExportMetadata.record(:table => @export_obj.table_name,
                             :database => @export_obj.source_schema,
+                            :export_to => load_to,
                             :export_id => @export_obj.id,
                             :filepath => filepath,
+                            :store_in => @export_obj.s3_path,
                             :incremental_col => @export_obj.incremental_column) do |metadata|
 
         prepare metadata
 
-        case @export_obj.export_type?
-        when :new
+        if @export_obj.export_type? == :new && load_to == "mysql"
           on_failure_state_trans(metadata, "new") # If failed, go back to new
           on_export_success(metadata)
           initial_export metadata
-
-        when :incremental
+        elsif @export_obj.export_type? == :incremental or load_to == "vertica"
           on_failure_state_trans(metadata, "failed") # Set state trans on failure
           on_export_success(metadata)
           incremental_export metadata
@@ -35,7 +35,15 @@ module Myreplicator
         
       end # metadata
     end
-    
+
+    def load_to
+      if @export_obj.export_to == "vertica"
+        return "vertica"
+      else
+        return "mysql"
+      end
+    end
+
     ##
     # Setups SSH connection to remote host
     ##
@@ -65,12 +73,10 @@ module Myreplicator
       metadata.export_type = "initial"
       max_value = @export_obj.max_value if @export_obj.incremental_export?
       cmd = initial_mysqldump_cmd
-
       exporting_state_trans # mark exporting
 
       puts "Exporting..."
       result = execute_export(cmd, metadata)
-      
       check_result(result, 0)
 
       @export_obj.update_max_val(max_value) if @export_obj.incremental_export?
@@ -78,10 +84,19 @@ module Myreplicator
 
     def initial_mysqldump_cmd
       flags = ["create-options", "single-transaction"]       
-      cmd = SqlCommands.mysqldump(:db => @export_obj.source_schema,
-                                  :flags => flags,
-                                  :filepath => filepath,
-                                  :table_name => @export_obj.table_name)     
+      cmd = ""
+      # Mysql - Mysql Export
+      if @export_obj.export_to == "destination_db"
+        cmd = SqlCommands.mysqldump(:db => @export_obj.source_schema,
+                                    :flags => flags,
+                                    :filepath => filepath,
+                                    :table_name => @export_obj.table_name)     
+      else # Other destinations
+        cmd = SqlCommands.mysql_export_outfile(:db => @export_obj.source_schema,
+                                               :filepath => filepath,
+                                               :table => @export_obj.table_name)
+      end
+      
       return cmd
     end
     
@@ -118,6 +133,7 @@ module Myreplicator
       cmd = SqlCommands.mysql_export(:db => @export_obj.source_schema,
                                      :filepath => filepath,
                                      :sql => sql)
+      
       return cmd
     end
 
@@ -125,27 +141,85 @@ module Myreplicator
     # Exports table incrementally, similar to incremental_export method
     # Dumps file in tmp directory specified in myreplicator.yml
     # Note that directory needs 777 permissions for mysql to be able to export the file
-    # Uses ;~; as the delimiter and new line for lines
+    # Uses \\0 as the delimiter and new line for lines
     ##
 
     def incremental_export_into_outfile metadata
-      max_value = @export_obj.max_value
-      metadata.export_type = "incremental_outfile"
-
-      @export_obj.update_max_val if @export_obj.max_incremental_value.blank?   
-
-      cmd = SqlCommands.mysql_export_outfile(:db => @export_obj.source_schema,
-                                             :table => @export_obj.table_name,
-                                             :filepath => filepath,
-                                             :incremental_col => @export_obj.incremental_column,
-                                             :incremental_col_type => @export_obj.incremental_column_type,
-                                             :incremental_val => @export_obj.max_incremental_value)      
-      exporting_state_trans
-      puts "Exporting..."
-      result = execute_export(cmd, metadata)
-      check_result(result, 0)
-      max_value = incremental_export_into_outfile(metadata)
-      return max_value
+      unless @export_obj.is_running?
+        max_value = @export_obj.max_value
+        metadata.export_type = "incremental_outfile"        
+        @export_obj.update_max_val if @export_obj.max_incremental_value.blank?   
+      
+        options = {
+          :db => @export_obj.source_schema,
+          :table => @export_obj.table_name,
+          :filepath => filepath,
+          :destination_schema => @export_obj.destination_schema}
+
+        unless schema_changed?(options)[:changed]
+          options[:incremental_col] = @export_obj.incremental_column
+          options[:incremental_col_type] = @export_obj.incremental_column_type
+          options[:incremental_val] = @export_obj.max_incremental_value
+        end
+
+        cmd = SqlCommands.mysql_export_outfile(options)      
+        exporting_state_trans
+        puts "Exporting..."
+        result = execute_export(cmd, metadata)
+        check_result(result, 0)
+        metadata.incremental_val = max_value # store max val in metadata
+        @export_obj.update_max_val(max_value) # update max value if export was successful
+      end
+      return false
+    end
+
+    def self.compare_schemas vertica_schema, mysql_schema
+      if vertica_schema.size != mysql_schema.size
+        return {:changed => true, :mysql_schema => mysql_schema, :vertica_schema => vertica_schema,:new => false}
+      else
+        index = 0
+        while index < vertica_schema.size
+          # puts vertica_schema.rows[index][:column_name]  + " " + mysql_schema[index]["column_name"]
+          # check for column name
+          if vertica_schema.rows[index][:column_name] != mysql_schema[index]["column_name"]
+            return true
+          end
+  
+          # puts vertica_schema.rows[index][:data_type]  + " " + VerticaTypes.convert(mysql_schema[index]["data_type"],mysql_schema[index]["column_type"])
+          # check for column's data type
+          if (vertica_schema.rows[index][:data_type] != VerticaTypes.convert(mysql_schema[index]["data_type"],mysql_schema[index]["column_type"]) and vertica_schema.rows[index][:data_type] != "timestamp")
+            return true
+          end
+          # and others ?? (PRIMARY, DEFAULT NULL, etc.)
+          index += 1
+        end
+      end
+      return false      
+    end
+    
+    def self.schema_changed? options
+      puts options
+      mysql_schema = Loader.mysql_table_definition(options)
+      vertica_schema = VerticaLoader.destination_table_vertica(options)
+
+      # empty result set from vertica means table does not exist 
+      unless vertica_schema.size > 0 
+        return {:changed => true, :mysql_schema => mysql_schema, :new => true}
+      end
+      
+      # compare two schemas
+      mysql_schema_simple_form = []
+      mysql_schema.each(:as => :hash) do |row|
+        mysql_schema_simple_form << row  
+      end
+      
+      mysql_schema_2 = mysql_schema_simple_form
+      if compare_schemas(vertica_schema, mysql_schema_2)
+        result =  {:changed => true, :mysql_schema => mysql_schema, :vertica_schema => vertica_schema,:new => false}
+      else
+        result =  {:changed => false}
+      end
+      return result
     end
 
     ##

data/lib/exporter/sql_commands.rb

@@ -36,7 +36,7 @@ module Myreplicator
       # cmd += "--tab=#{options[:filepath]} "
       # cmd += "--fields-enclosed-by=\'\"\' "
       # cmd += "--fields-escaped-by=\'\\\\\' "
-
+        
       return cmd
     end
 
@@ -80,16 +80,17 @@ module Myreplicator
       options = args.extract_options!
       options.reverse_merge! :flags => []
       db = options[:db]
+      
       # Database host when ssh'ed into the db server
       
       db_host = "127.0.0.1" 
-
+      
       if !ssh_configs(db)["ssh_db_host"].blank? 
         db_host =  ssh_configs(db)["ssh_db_host"]
       elsif !db_configs(db)["host"].blank?
-        db_host += db_configs(db)["host"]
+        db_host = db_configs(db)["host"]
       end
-
+      
       flags = ""
 
       self.mysql_flags.each_pair do |flag, value|
@@ -120,11 +121,11 @@ module Myreplicator
     def self.get_outfile_sql options 
       sql = "SELECT * INTO OUTFILE '#{options[:filepath]}' " 
       
-      sql += " FIELDS TERMINATED BY ';~;' OPTIONALLY ENCLOSED BY '\\\"'  LINES TERMINATED BY '\\n'"
+      sql += " FIELDS TERMINATED BY '\\0' OPTIONALLY ENCLOSED BY '\\\"'  LINES TERMINATED BY '\\n'"
       
       sql += "FROM #{options[:db]}.#{options[:table]} "
 
-      if options[:incremental_col] && options[:incremental_val]
+      if options[:incremental_col] && !options[:incremental_val].blank?
         if options[:incremental_col_type] == "datetime"
           sql += "WHERE #{options[:incremental_col]} >= '#{options[:incremental_val]}'"
         else
@@ -159,8 +160,6 @@ module Myreplicator
         end
       end
     
-      # BOB : You always build the host,password,username part of the command
-      # Seems like this should be in a function somewhere 
       cmd = Myreplicator.mysql
       cmd += "#{flags} -u#{db_configs(db)["username"]} -p#{db_configs(db)["password"]} " 
       cmd += "-h#{db_host} " if db_configs(db)["host"].blank?
@@ -189,7 +188,7 @@ module Myreplicator
       options = args.extract_options!
       sql = "SELECT * FROM #{options[:db]}.#{options[:table]} " 
       
-      if options[:incremental_col] && options[:incremental_val]
+      if options[:incremental_col] && !options[:incremental_val].blank?
         if options[:incremental_col_type] == "datetime"
           sql += "WHERE #{options[:incremental_col]} >= '#{options[:incremental_val]}'"
         else

data/lib/exporter.rb

@@ -1,3 +1,4 @@
 Dir["#{File.expand_path(File.dirname(__FILE__))}/exporter/*.rb"].each { | f | require File.expand_path(f) }
 Dir["#{File.expand_path(File.dirname(__FILE__))}/transporter/*.rb"].each { | f | require File.expand_path(f) }
 Dir["#{File.expand_path(File.dirname(__FILE__))}/loader/*.rb"].each { | f | require File.expand_path(f) }
+Dir["#{File.expand_path(File.dirname(__FILE__))}/loader/vertica/*.rb"].each { | f | require File.expand_path(f) }

data/lib/loader/loader.rb

@@ -177,24 +177,31 @@ module Myreplicator
       metadata.zipped = false
       
       options = {:table_name => exp.table_name, :db => exp.destination_schema,
-        :filepath => metadata.destination_filepath(tmp_dir)}
+        :filepath => metadata.destination_filepath(tmp_dir), :source_schema => exp.source_schema}
       
       if metadata.export_type == "incremental_outfile"
-        options[:fields_terminated_by] = ";~;"
+        options[:fields_terminated_by] = "\\0"
         options[:lines_terminated_by] = "\\n"
       end
       
-      cmd = ImportSql.load_data_infile(options)
-      
-      puts cmd
-      
-      result = `#{cmd}` # execute
-      
-      unless result.nil?
-        if result.size > 0
-          raise Exceptions::LoaderError.new("Incremental Load #{metadata.filename} Failed!\n#{result}") 
+      case metadata.export_to 
+      when "vertica"
+        options = {:table_name => exp.table_name, :db => ActiveRecord::Base.configurations["vertica"]["database"],
+          :filepath => metadata.destination_filepath(tmp_dir), :source_schema => exp.source_schema, :export_id => metadata.export_id}
+        options[:destination_schema] = exp.destination_schema
+        Kernel.p "===== LOAD TO VERTICA ====="
+        Kernel.p options
+        Myreplicator::VerticaLoader.load options
+      when "mysql"
+        cmd = ImportSql.load_data_infile(options)
+        puts cmd
+        result = `#{cmd}` # execute
+        unless result.nil?
+          if result.size > 0
+            raise Exceptions::LoaderError.new("Incremental Load #{metadata.filename} Failed!\n#{result}") 
+          end
         end
-      end
+      end #case  
     end
 
     ##
@@ -252,5 +259,18 @@ module Myreplicator
       return files
     end
 
+    def self.mysql_table_definition options
+      Kernel.p options
+      sql = "SELECT table_schema, table_name, column_name, is_nullable, data_type, column_type, column_key "
+      sql += "FROM INFORMATION_SCHEMA.COLUMNS where table_name = '#{options[:table]}' "
+      sql += "and table_schema = '#{options[:source_schema]}';"
+      
+      puts sql
+      
+      desc = DB.exec_sql(options[:source_schema], sql)
+      puts desc
+      return desc
+    end
+
   end
 end

data/lib/loader/vertica/source_db.rb

@@ -0,0 +1,30 @@
+module Myreplicator
+  class DB < ActiveRecord::Base
+      
+    def self.connect db
+      establish_connection(ActiveRecord::Base.configurations[db])
+    end
+  
+    ##
+    # Returns tables as an Array
+    # releases the connection
+    ##
+    def self.get_tables(db)
+      tables = []
+      begin
+        self.connect(db)
+        tables = self.connection.tables
+        self.connection_pool.release_connection
+      rescue Mysql2::Error => e
+        puts "Connection to #{db} Failed!"
+        puts e.message
+      end
+      return tables
+    end
+  
+    def self.exec_sql source_db,sql
+      DB.connect(source_db)
+      return DB.connection.execute(sql)
+    end
+  end
+end

data/lib/loader/vertica/source_db.rb~

@@ -0,0 +1,30 @@
+module VerticaUtils
+  class SourceDb < ActiveRecord::Base
+      
+    def self.connect db
+      establish_connection(ActiveRecord::Base.configurations[db])
+    end
+  
+    ##
+    # Returns tables as an Array
+    # releases the connection
+    ##
+    def self.get_tables(db)
+      tables = []
+      begin
+        self.connect(db)
+        tables = self.connection.tables
+        self.connection_pool.release_connection
+      rescue Mysql2::Error => e
+        puts "Connection to #{db} Failed!"
+        puts e.message
+      end
+      return tables
+    end
+  
+    def self.exec_sql source_db,sql
+      SourceDb.connect(source_db)
+      return SourceDb.connection.execute(sql)
+    end
+  end
+end

data/lib/loader/vertica/types.rb

@@ -0,0 +1,44 @@
+module Myreplicator
+  class VerticaTypes
+  
+    def self.convert type, col_type
+      if mysql_vertica_conversion[type].blank?
+        return col_type
+      else
+        return mysql_vertica_conversion[type]
+      end
+    end
+
+    def self.mysql_vertica_conversion
+      map = {
+        "int" => "int",
+        "integer" => "int",
+        "int8" => "int",
+        "smallint" => "int",
+        "bigint" => "int",
+        "tinyint" => "int",
+        "numeric" => "int",
+        "text" => "VARCHAR(65000)",
+        "mediumtext" => "VARCHAR(65000)",
+        "bit" => "binary",
+        "longtext" => "VARCHAR(65000)",
+        "float" => "decimal"
+      }
+    end
+
+    def self.convert_key key
+      map = {
+        "UNI" => "UNIQUE",
+      " MUL" => "", 
+        "PRI" => "PRIMARY KEY"
+      }
+
+      if map[key].blank?
+        return ""
+      else
+        return map[key]
+      end
+    end
+
+  end
+end