myreplicator 0.0.16 → 0.0.17

data/app/views/myreplicator/exports/_form.html.erb

@@ -3,7 +3,6 @@
 export_to = ["destination_db","s3","both"]
 export_type = ["incremental","fulldump"]
 %>
-
 <%= form_for(@export) do |f| %>
   <% if @export.errors.any? %>
     <div id="error_explanation">
@@ -18,11 +17,11 @@ export_type = ["incremental","fulldump"]
   <% end %>
   <div class="form-section">
 <label>Source Schema</label>
-<%= f.text_field :source_schema %>
+<%= f.select :source_schema, @dbs %>
 <label>Destination Schema</label>
-<%= f.text_field :destination_schema %>
+<%= f.select :destination_schema, @dbs %>
 <label>Table Name</label>
-<%= f.text_field :table_name %>
+<%= f.select :table_name, [] %>
 <label>Incremental Column</label>
 <%= f.text_field :incremental_column %>
 <label>Maximum Incremental Value</label>
@@ -93,12 +92,29 @@ export_type = ["incremental","fulldump"]
 <%= javascript_include_tag "myreplicator/chosen.jquery.min" %>
 <%= javascript_include_tag "myreplicator/cronwtf.min" %>
 <script>
+var dbs = {
+<% @tables.each do |key,values| %>
+  "<%= key %>":[
+    <% values.each do |table| %>
+      "<%= table %>",
+    <% end %>
+  ],
+<% end %>
+}
 $(function(){
   CronUI.translate();
   $(".chosen").chosen();
+  $("#export_destination_schema,#export_table_name").chosen();
   $(".cron").chosen().change(function(){CronUI.translate()});
+  $("#export_source_schema").chosen().change(function(){
+    exportSchemaSelect($(this).val())
+  });
+  exportSchemaSelect($("#export_source_schema").val());
+<%- if @edit -%>
+  editInit();
+<%- end -%>
+})
 
-  <%- if @edit -%>
 function editInit(){
   var cron = "<%= @export.cron %>";
   var displays = ["#min","#hour","#day","#month","#dow"]; 
@@ -108,10 +124,18 @@ function editInit(){
     $(displays[i]).val(cron_vals[i]).trigger("liszt:updated");
   }
   CronUI.translate();
+  $("#export_table_name").val("<%= @export.table_name %>").trigger("liszt:updated");
 }
-editInit();
-  <%- end -%>
 
-
-})
+function exportSchemaSelect(val){
+    var target = $("#export_table_name");
+    var tables = dbs[val];
+    var l = tables.length;
+    var options = "";
+    for(i=0;i<l;i++){
+      options += "<option value='"+tables[i]+"'>"+tables[i]+"</option>";
+    }
+    target.val('').find("option").remove().trigger("liszt:updated");
+    target.append(options).trigger("liszt:updated");
+}
 </script>

data/app/views/myreplicator/home/_home_menu.erb

@@ -1,5 +1,9 @@
+<% 
+  err = err_count 
+  run = run_count
+%>
 <div class="home-menu">
   <span>lists:</span>
-  <a href="<%= root_url %>" class="<% if @option == 'overview' %>on<% end %>">overview</a>
-  <a href="<%= errors_path %>" class="<% if @option == 'errors' %>on<% end %>">errors</a>
+  <a href="<%= root_url %>" class="<% if @option == 'overview' %>on<% end %> overview">overview <% if run > 0 %><span><%= run %></span><% end %></a>
+  <a href="<%= errors_path %>" class="<% if @option == 'errors' %>on<% end %>">errors <% if err > 0 %><span><%= err %></span><% end %></a>
 </div>

data/app/views/myreplicator/home/errors.html.erb

@@ -1,10 +1,20 @@
 <%= render :partial => 'home_menu' %>
 <% @exports.each do |ex| %>
 <div class="error-display">
-<span><em>table name:</em> <%= ex.table_name %></span>
-<span><em>source schema:</em><%=  ex.source_schema %></span>
-<span><em>updated at:</em><%= ex.updated_at.strftime("%Y-%m-%d %H:%M") %></span>
-<span><em>error:</em></span>
-<pre><%= ex.error %></pre>
+  <span><em>table name:</em> <%= ex.table_name %></span>
+  <span><em>source schema:</em><%=  ex.source_schema %></span>
+  <span><em>updated at:</em><%= ex.updated_at.strftime("%Y-%m-%d %H:%M") %></span>
+  <span><em>error:</em></span>
+  <pre><%= ex.error %></pre>
+</div>
+<% end %>
+<% @logs.each do |log| %>
+<div class="error-display">
+  <span><em>name:</em> <%= log.name %></span>
+  <span><em>file:</em> <%= log.file %></span>
+  <span><em>type:</em> <%= log.job_type %></span>
+  <span><em>started:</em> <%= log.started_at.strftime("%Y-%m-%d %H:%M:%S") %></span>
+  <span><em>error:</em></span>
+  <pre><%= log.error %></pre>
 </div>
 <% end %>

data/app/views/myreplicator/home/index.html.erb

@@ -1,48 +1,25 @@
 <%= render :partial => 'home_menu' %>
-<% states = @exports.group_by(&:state) %>
 <div id="overview-wrapper">
-  <div id="simple-wrapper">
-    <table id="states-list">
-      <thead>
-        <tr>
-          <th>State</th>
-          <th class="centered">Count</th>
-        </tr>
-      </thead>
-      <tbody>
-    <% states.each do |state, exports| %>
-      <tr>
-        <td><a href="#<%= state.downcase %>_state" class="state-toggle"><%= state %></a></td>
-        <td class="c"><%= exports.size %></td>
-      </tr>
-    <% end %>
-      </tbody>
-    </table>
-    <p class="hint">***click a state to view export list</p>
-  </div>
-  <ul id="states-tables">
-    <% states.each do |state, exports| %>
-    <li id="<%= state.downcase %>_state">
-      <h4><%= state %> Exports</h4>
-      <table class="data-grid">
-        <thead>
-          <tr>
-            <th>Table</th>
-            <th>Schema</th>
-            <th>Updated</th>
-          </tr>
-        </thead>
-        <tbody>
-          <% exports.sort{|a,b| b.updated_at<=>a.updated_at}.each do |ex| %>
-            <td><%= ex.table_name %></td>
-            <td><%= ex.source_schema %></td>
-            <td><%= ex.updated_at.strftime("%Y-%m-%d %H:%M") %></td>
-          <% end %>
-        </tbody>
-      </table>
-    </li>
-    <% end %>
-  </ul>
+<h6>Running tasks</h6>
+<table class="overview">
+  <% @logs.each do |log| %>
+  <tr>
+    <td>
+      <span class="name"><%= log.name %></span>
+      <span class="file"><%= log.file %></span>
+    </td>
+    <td><%= log.state %></td>
+    <td><%= log.job_type %></td>
+    <td><%= log.started_at.strftime("%Y-%m-%d %H:%M:%S") %></td>
+    <td><%= chronos(@now.to_i - log.started_at.to_i) %></td>
+  </tr>
+  <% end %>
+</table>
+
+
+
+
+
 </div>
 <script>
 jQuery(function(){

data/db/migrate/20121025191622_create_myreplicator_exports.rb

@@ -15,14 +15,8 @@ class CreateMyreplicatorExports < ActiveRecord::Migration
       t.text :error
       t.boolean :active, :default => true
       t.integer :exporter_pid
-      t.integer :transporter_pid
-      t.integer :loader_pid
       t.datetime :export_started_at, :default => nil
       t.datetime :export_finished_at, :default => nil
-      t.datetime :load_started_at, :default => nil
-      t.datetime :load_finished_at, :default => nil
-      t.datetime :transfer_started_at, :default => nil
-      t.datetime :transfer_finished_at, :default => nil
       t.timestamps
     end  
     add_index :myreplicator_exports, [:source_schema, :destination_schema, :table_name], :unique => true, :name => "unique_index"

data/db/migrate/20121212003652_create_myreplicator_logs.rb

@@ -0,0 +1,21 @@
+class CreateMyreplicatorLogs < ActiveRecord::Migration
+  def change
+    create_table :myreplicator_logs do |t|
+      t.integer :pid
+      t.string :job_type
+      t.string :name
+      t.string :file
+      t.string :state
+      t.string :thread_state
+      t.string :hostname
+      t.string :export_id
+      t.text :error
+      t.text :backtrace
+      t.string :guid
+      t.datetime :started_at, :default => nil
+      t.datetime :finished_at, :default => nil
+
+      t.timestamps
+    end
+  end
+end

data/lib/exporter/export_metadata.rb

@@ -32,6 +32,7 @@ module Myreplicator
       end
     end
     
+    # BOB : This only handles gzipped files, is that what you want?
     def filename
       name = filepath.split("/").last
       name = zipped ? "#{name}.gz" : name
@@ -44,7 +45,7 @@ module Myreplicator
 
     ##
     # Keeps track of the state of the export
-    # Store itself in a JSON file on exit
+    # Stores itself in a JSON file on exit
     ##
     def self.record *args
       options = args.extract_options!
@@ -160,6 +161,11 @@ module Myreplicator
       return path
     end
 
+    ##
+    # Writes Json to file using echo
+    # file is written to remote server via SSH
+    # Echo is used for writing the file
+    ##
     def store!
       cmd = "echo \"#{self.to_json.gsub("\"","\\\\\"")}\" > #{@filepath}.json"
       puts cmd
@@ -175,6 +181,7 @@ module Myreplicator
 
     def set_attributes options
       options.symbolize_keys!
+
       @export_time = options[:export_time] if options[:export_time]
       @table = options[:table] if options[:table]
       @database = options[:database] if options[:database]

data/lib/exporter/mysql_exporter.rb

@@ -31,9 +31,12 @@ module Myreplicator
             metadata.state = "export_completed"
             wrapup metadata
           end
+
         elsif !is_running?
           # local max value for incremental export
+
           max_value = incremental_export(metadata)
+          #max_value = incremental_export_into_outfile(metadata)
 
           metadata.incremental_val = max_value # store max val in metadata
 
@@ -128,9 +131,38 @@ module Myreplicator
       puts "Exporting..."
       result = execute_export(cmd, metadata)
       check_result(result, 0)
+
+      return max_value
     end
 
 
+    ##
+    # Exports table incrementally, similar to incremental_export method
+    # Dumps file in tmp directory specified in myreplicator.yml
+    # Note that directory needs 777 permissions for mysql to be able to export the file
+    # Uses ;~; as the delimiter and new line for lines
+    ##
+
+    def incremental_export_into_outfile metadata
+      max_value = @export_obj.max_value
+      @export_obj.update_max_val if @export_obj.max_incremental_value.blank?   
+
+      cmd = SqlCommands.mysql_export_outfile(:db => @export_obj.source_schema,
+                                             :table => @export_obj.table_name,
+                                             :filepath => filepath,
+                                             :incremental_col => @export_obj.incremental_column,
+                                             :incremental_col_type => @export_obj.incremental_column_type,
+                                             :incremental_val => @export_obj.max_incremental_value)      
+
+      metadata.export_type = "incremental_outfile"
+      update_export(:state => "exporting", :export_started_at => Time.now, :exporter_pid => Process.pid)
+      puts "Exporting..."
+      result = execute_export(cmd, metadata)
+      check_result(result, 0)
+
+      return max_value
+    end
+
     ##
     # Completes an export process
     # Zips files, updates states etc

data/lib/exporter/sql_commands.rb

@@ -34,14 +34,25 @@ module Myreplicator
       return cmd
     end
 
+    ##
+    # Db configs for active record connection
+    ## 
+
     def self.db_configs db
       ActiveRecord::Base.configurations[db]
     end
 
+    ##
+    # Configs needed for SSH connection to source server
+    ##
+
     def self.ssh_configs db
       Myreplicator.configs[db]
     end
 
+    ##
+    # Default dump flags
+    ## 
     def self.dump_flags
       {"add-locks" => true,
         "compact" => false,
@@ -55,6 +66,10 @@ module Myreplicator
       }
     end
 
+    ##
+    # Mysql exports using -e flag
+    ## 
+
     def self.mysql_export *args
       options = args.extract_options!
       options.reverse_merge! :flags => []
@@ -83,6 +98,69 @@ module Myreplicator
       return cmd
     end
 
+    ##
+    # Mysql export data into outfile option
+    # Provided for tables that need special delimiters
+    ##
+    
+    def self.get_outfile_sql options 
+      sql = "SELECT * INTO OUTFILE '#{options[:filepath]}' " 
+      
+      sql += " FIELDS TERMINATED BY ';~;' OPTIONALLY ENCLOSED BY '\\\"'  LINES TERMINATED BY '\\n'"
+      
+      sql += "FROM #{options[:db]}.#{options[:table]} "
+
+      if options[:incremental_col] && options[:incremental_val]
+        if options[:incremental_col_type] == "datetime"
+          sql += "WHERE #{options[:incremental_col]} >= '#{options[:incremental_val]}'"
+        else
+          sql += "WHERE #{options[:incremental_col]} >= #{options[:incremental_val]}"
+        end
+      end
+
+      return sql
+    end
+
+    ##
+    # Export using outfile
+    # ;~; delimited
+    # terminated by newline 
+    # Location of the output file needs to have 777 perms
+    ##
+    def self.mysql_export_outfile *args
+      options = args.extract_options!
+      options.reverse_merge! :flags => []
+      db = options[:db]
+
+      # Database host when ssh'ed into the db server
+      db_host = ssh_configs(db)["ssh_db_host"].nil? ? "127.0.0.1" : ssh_configs(db)["ssh_db_host"]
+
+      flags = ""
+
+      self.mysql_flags.each_pair do |flag, value|
+        if options[:flags].include? flag
+          flags += " --#{flag} "
+        elsif value
+          flags += " --#{flag} "
+        end
+      end
+    
+      # BOB : You always build the host,password,username part of the command
+      # Seems like this should be in a function somewhere 
+      cmd = Myreplicator.mysql
+      cmd += "#{flags} -u#{db_configs(db)["username"]} -p#{db_configs(db)["password"]} " 
+      cmd += "-h#{db_host} " if db_configs(db)["host"].blank?
+      cmd += db_configs(db)["port"].blank? ? "-P3306 " : "-P#{db_configs(db)["port"]} "
+      cmd += "--execute=\"#{get_outfile_sql(options)}\" "
+      
+      puts cmd
+
+      return cmd
+    end
+
+    ##
+    # Default flags for mysql export
+    ## 
     def self.mysql_flags
       {"column-names" => false,
         "quick" => true,
@@ -90,6 +168,9 @@ module Myreplicator
       }    
     end
 
+    ##
+    # Builds SQL needed for incremental exports
+    ##
     def self.export_sql *args
       options = args.extract_options!
       sql = "SELECT * FROM #{options[:db]}.#{options[:table]} " 
@@ -105,6 +186,10 @@ module Myreplicator
       return sql
     end
 
+    ##
+    # Gets the Maximum value for the incremental 
+    # column of the export job
+    ##
     def self.max_value_sql *args
       options = args.extract_options!
       sql = ""
@@ -118,9 +203,5 @@ module Myreplicator
       return sql
     end
 
-    def self.mysql_export_outfile
-      
-    end
-
   end
 end

data/lib/loader/loader.rb

@@ -3,6 +3,8 @@ require "exporter"
 module Myreplicator
   class Loader
     
+    @queue = :myreplicator_load # Provided for Resque
+    
     def initialize *args
       options = args.extract_options!
     end
@@ -11,10 +13,22 @@ module Myreplicator
       @tmp_dir ||= File.join(Myreplicator.app_root,"tmp", "myreplicator")
     end
 
-    def load
+    ##
+    # Main method provided for resque
+    ##
+    def self.perform
+      load # Kick off the load process
+    end
+
+    ##
+    # Kicks off all initial loads first and then all incrementals
+    # Looks at metadata files stored locally
+    ##
+    def self.load
       initials = []
       incrementals = []
 
+      # Read all metadata files
       metadata_files.each do |metadata|
         if metadata.export_type == "initial"
           initials << metadata
@@ -22,16 +36,32 @@ module Myreplicator
           incrementals << metadata
         end
       end
-      
+
+      # Load all new tables
       initials.each do |metadata| 
         puts metadata.table
-        initial_load metadata
+
+        Log.run(:job_type => "loader", 
+                :name => "initial_import", 
+                :file => metadata.filename, 
+                :export_id => metadata.export_id) do |log|
+          
+          initial_load metadata
+        end
+
         cleanup metadata
       end
 
+      # Load all incremental files
       incrementals.each do |metadata|
         puts metadata.table
-        incremental_load metadata
+        Log.run(:job_type => "loader", 
+                :name => "incremental_import", 
+                :file => metadata.filename, 
+                :export_id => metadata.export_id) do |log|
+
+          incremental_load metadata
+        end
         cleanup metadata
       end
     end
@@ -47,12 +77,14 @@ module Myreplicator
       cmd = ImportSql.initial_load(:db => exp.destination_schema,
                                    :filepath => metadata.destination_filepath(tmp_dir))      
       puts cmd
+
       result = `#{cmd}` # execute
       unless result.nil?
         if result.size > 0
           raise Exceptions::LoaderError.new("Initial Load #{metadata.filename} Failed!\n#{result}") 
         end
       end
+      
     end
 
     ##
@@ -63,12 +95,21 @@ module Myreplicator
       exp = Export.find(metadata.export_id)
       unzip(metadata.filename)
       metadata.zipped = false
+      
+      options = {:table_name => exp.table_name, :db => exp.destination_schema,
+        :filepath => metadata.destination_filepath(tmp_dir)}
+
+      if metadata.export_type == "incremental_outfile"
+        options[:fields_terminated_by] = ";~;"
+        options[:lines_terminated_by] = "\\n"
+      end
+
+      cmd = ImportSql.load_data_infile(options)
 
-      cmd = ImportSql.load_data_infile(:table_name => exp.table_name, 
-                                       :db => exp.destination_schema,
-                                       :filepath => metadata.destination_filepath(tmp_dir))
       puts cmd
+      
       result = `#{cmd}` # execute
+      
       unless result.nil?
         if result.size > 0
           raise Exceptions::LoaderError.new("Incremental Load #{metadata.filename} Failed!\n#{result}")