myreplicator 0.0.1

data/app/views/myreplicator/exports/index.html.erb

@@ -0,0 +1,56 @@
+<%= link_to content_tag(:span, 'add export'), new_export_path, :class => "btn right main add" %>
+<h2>Exports</h2>
+<%= will_paginate @exports , :previous_label => '<', :next_label => '>', :inner_window => 2, :outer_window => 0%>
+<% if @exports.total_entries > @exports.per_page %>
+<div id="pagination-display-message">Displaying <%= @exports.offset + 1 %> - <%= @exports.offset + @exports.length %> of <%= pluralize(@exports.total_entries, 'exports') %></div>
+<% end %>
+<div class="table-wrapper">
+  <table class="data-grid">
+    <thead>
+      <tr>
+       <th colspan="2"><%= sortable "source_schema" %></th>
+       <th><%= sortable "destination_schema" %></th>
+       <th><%= sortable "table_name" %></th>
+       <th><%= sortable "incremental_column" %></th>
+       <th><%= sortable "max_incremental_value" %></th>
+       <th><%= sortable "export_to", "Export Desitination" %></th>
+       <th><%= sortable "export_type" %></th> 
+       <th><%= sortable "s3_path" %></th>
+       <th><%= sortable "cron" %></th> 
+       <th class="center">Actions</th>
+      </tr>
+    </thead>
+    <tbody>
+    <% @exports.each do |export| %>
+      <tr>
+        <td class="state"><span class="status <% if export.active %>active<% else %>inactive<% end %>" title="<% if export.active %>Active<% else %>Inactive<% end %>"></span></td> 
+        <td class="source"><%= export.source_schema %></td>
+        <td><%= export.destination_schema %></td>
+        <td><%= export.table_name %></td>
+        <td><%= export.incremental_column %></td>
+        <td><%= export.max_incremental_value %></td>
+        <td><%= export.export_to %></td>
+        <td><%= export.export_type %></td> 
+        <td><%= export.s3_path %></td>
+        <td><span class="cron" data-cron="<%= export.cron %>" title=""><%= export.cron %></span></td>     
+        <td>
+          <%= link_to 'review', export, :class=> 'action view' %>
+          <%= link_to 'edit', edit_export_path(export), :class => 'action edit' %>
+          <%= link_to 'destroy', export, method: :delete, data: { confirm: 'Are you sure?' }, :class=> 'action delete' %>
+        </td>
+      </tr>
+    <% end %>
+    </tbody>
+  </table>
+</div>
+<script>
+$(function(){
+  $.each($("span.cron"),function(i){
+    var span = $(this)
+    var trans = String(CronWTF.parse(span.data("cron")));
+    span.attr("title", trans)
+  })
+  $("span.status").tipTip();
+  $("span.cron").tipTip();
+})
+</script>

data/app/views/myreplicator/exports/new.html.erb

@@ -0,0 +1,2 @@
+<h2>New Export</h2>
+<%= render 'form' %>

data/app/views/myreplicator/exports/show.html.erb

@@ -0,0 +1,28 @@
+<%= link_to content_tag(:span, 'edit export'), edit_export_path(@export), :class =>'btn right main edit' %>
+<h2>Review Export</h2>
+<ul id="export-review">
+  <% @export.attributes.sort{|a,b| a[0] <=> b[0]}.each do |key,val| %>
+    <% type = @export.column_for_attribute(key).type %>
+    <li><span><%= key.gsub("_"," ") %> : </span>
+      <%- if type == :datetime -%>
+        <%- unless val.blank? -%>
+          <%= val.strftime("%B #{val.day.ordinalize}, %Y %I:%M %p") -%>
+        <%- end -%>
+      <%- else -%>
+        <% if key == 'cron' %>
+          <em class="cron" data-cron="<%= val %>" title=""><%= val %></em>
+        <%- else -%>
+          <%= val -%>
+        <%- end -%>
+      <%- end -%>
+    </li>
+  <% end %>
+</ul>
+<script>
+$(function(){
+  var span = $("em.cron")
+  var trans = String(CronWTF.parse(span.data("cron")));
+  span.attr("title", trans)
+  $("em.cron").tipTip();
+})
+</script>

data/app/views/myreplicator/home/_home_menu.erb

@@ -0,0 +1,5 @@
+<div class="home-menu">
+  <span>lists:</span>
+  <a href="<%= root_url %>" class="<% if @option == 'overview' %>on<% end %>">overview</a>
+  <a href="<%= errors_path %>" class="<% if @option == 'errors' %>on<% end %>">errors</a>
+</div>

data/app/views/myreplicator/home/errors.html.erb

@@ -0,0 +1,10 @@
+<%= render :partial => 'home_menu' %>
+<% @exports.each do |ex| %>
+<div class="error-display">
+<span><em>table name:</em> <%= ex.table_name %></span>
+<span><em>source schema:</em><%=  ex.source_schema %></span>
+<span><em>updated at:</em><%= ex.updated_at.strftime("%Y-%m-%d %H:%M") %></span>
+<span><em>error:</em></span>
+<pre><%= ex.error %></pre>
+</div>
+<% end %>

data/app/views/myreplicator/home/index.html.erb

@@ -0,0 +1,58 @@
+<%= render :partial => 'home_menu' %>
+<% states = @exports.group_by(&:state) %>
+<div id="overview-wrapper">
+  <div id="simple-wrapper">
+    <table id="states-list">
+      <thead>
+        <tr>
+          <th>State</th>
+          <th class="centered">Count</th>
+        </tr>
+      </thead>
+      <tbody>
+    <% states.each do |state, exports| %>
+      <tr>
+        <td><a href="#<%= state.downcase %>_state" class="state-toggle"><%= state %></a></td>
+        <td class="c"><%= exports.size %></td>
+      </tr>
+    <% end %>
+      </tbody>
+    </table>
+    <p class="hint">***click a state to view export list</p>
+  </div>
+  <ul id="states-tables">
+    <% states.each do |state, exports| %>
+    <li id="<%= state.downcase %>_state">
+      <h4><%= state %> Exports</h4>
+      <table class="data-grid">
+        <thead>
+          <tr>
+            <th>Table</th>
+            <th>Schema</th>
+            <th>Updated</th>
+          </tr>
+        </thead>
+        <tbody>
+          <% exports.sort{|a,b| b.updated_at<=>a.updated_at}.each do |ex| %>
+            <td><%= ex.table_name %></td>
+            <td><%= ex.source_schema %></td>
+            <td><%= ex.updated_at.strftime("%Y-%m-%d %H:%M") %></td>
+          <% end %>
+        </tbody>
+      </table>
+    </li>
+    <% end %>
+  </ul>
+</div>
+<script>
+jQuery(function(){
+  var toggles = jQuery("a.state-toggle");
+  toggles.click(function(e){
+    e.preventDefault();
+    var link = $(this);
+    toggles.removeClass("on");
+    link.addClass("on");
+    jQuery(link.attr('href')).fadeIn().siblings("li").hide();
+  })
+})
+</script>

data/config/routes.rb

@@ -0,0 +1,6 @@
+Myreplicator::Engine.routes.draw do
+  resources :exports
+
+  root :to => "home#index"
+  match '/errors', :to => "home#errors", :as => 'errors'
+end

data/db/migrate/20121025191622_create_myreplicator_exports.rb

@@ -0,0 +1,35 @@
+class CreateMyreplicatorExports < ActiveRecord::Migration
+  def change
+    create_table :myreplicator_exports do |t|
+      t.string :source_schema
+      t.string :destination_schema
+      t.string :table_name
+      t.string :incremental_column
+      t.string :max_incremental_value
+      t.string :incremental_column_type
+      t.string :export_to, :default => "destination_db"
+      t.string :export_type, :default => "incremental"
+      t.string :s3_path
+      t.string :cron
+      t.string :state, :default => "new"
+      t.text :error
+      t.boolean :active, :default => true
+      t.integer :exporter_pid
+      t.integer :transporter_pid
+      t.integer :loader_pid
+      t.datetime :export_started_at, :default => nil
+      t.datetime :export_finished_at, :default => nil
+      t.datetime :load_started_at, :default => nil
+      t.datetime :load_finished_at, :default => nil
+      t.datetime :transfer_started_at, :default => nil
+      t.datetime :transfer_finished_at, :default => nil
+      t.timestamps
+    end  
+    add_index :myreplicator_exports, [:source_schema, :destination_schema, :table_name], :unique => true, :name => "unique_index"
+  end
+
+  def self.down
+    drop_table :myreplicator_exports
+  end
+  
+end

data/lib/configuration.rb

@@ -0,0 +1,25 @@
+require "myreplicator"
+
+module Myreplicator
+  ##
+  # Configuration class for avoiding load the yml file everytime and cleaner config settings
+  ##
+  class Configuration
+    attr_accessor :tmp_path, :mysqldump, :mysql
+    
+    yml = YAML.load(File.read("#{Myreplicator.app_root}/config/myreplicator.yml"))
+    Kernel.p yml
+    Kernel.p yml["myreplicator"]["tmp_path"]
+    @@tmp_path = yml["myreplicator"]["tmp_path"]
+    @@mysql = yml["myreplicator"]["mysql"]
+    @@mysqldump = yml["myreplicator"]["mysqldump"]
+  end
+
+  def self.config(&block)
+    @@config ||= Myreplicator::Configuration.new
+
+    yield @@config if block
+
+    return @@config
+  end
+end

data/lib/exporter/export_metadata.rb

@@ -0,0 +1,198 @@
+require 'json'
+
+module Myreplicator
+  class ExportMetadata
+
+    attr_accessor(:export_time, 
+                  :export_finished_at, 
+                  :table, 
+                  :database, 
+                  :state, 
+                  :incremental_col, 
+                  :export_id, 
+                  :incremental_val,
+                  :ssh,
+                  :export_type,
+                  :on_duplicate,
+                  :filepath,
+                  :zipped,
+                  :error)
+
+    attr_reader :failure_callbacks
+    attr_reader :success_callbacks
+    attr_reader :ensure_callbacks
+    attr_reader :ignore_callbacks
+
+    def initialize *args
+      options = args.extract_options!
+      if options[:metadata_path] 
+        load options[:metadata_path] 
+      else
+        set_attributes options
+      end
+    end
+    
+    def filename
+      name = filepath.split("/").last
+      name = zipped ? "#{name}.gz" : name
+      return name     
+    end
+
+    def destination_filepath tmp_dir
+      File.join(tmp_dir, filename)
+    end
+
+    ##
+    # Keeps track of the state of the export
+    # Store itself in a JSON file on exit
+    ##
+    def self.record *args
+      options = args.extract_options!
+      options.reverse_merge!(:export_time => Time.now,
+                             :state => "exporting")
+      begin
+        metadata = ExportMetadata.new 
+        metadata.set_attributes options
+
+        yield metadata
+
+        metadata.run_success_callbacks
+
+      rescue Exceptions::ExportError => e
+        metadata.state = "failed"     
+        metadata.error =  "#{e.message}\n#{e.backtrace}"
+        metadata.run_failure_callbacks
+
+      rescue Exceptions::ExportIgnored => e
+        metadata.state = "ignored"
+        metadata.run_ignore_callbacks
+        metadata.filepath = metadata.filepath + ".ignored"
+
+      ensure
+        metadata.export_finished_at = Time.now
+        metadata.state = "failed" if metadata.state == "exporting"
+        metadata.store!
+        metadata.ssh.close
+
+        metadata.run_ensure_callbacks
+      end
+    end
+
+    # Add a callback to run on failure of the 
+    # export
+    def on_failure *args, &block
+      if block_given?
+        @failure_callbacks << block
+      else
+        @failure_callbacks << args.shift
+      end
+    end
+
+    # Adds a callback that runs if the
+    # export is already running
+    def on_ignore *args, &block
+      if block_given?
+        @ignore_callbacks << block
+      else
+        @ignore_callbacks << args.shift
+      end
+    end
+
+    # Adds a callback that runs if the
+    # export is completed successfully
+    def on_success *args, &block
+      if block_given?
+        @success_callbacks << block
+      else
+        @success_callbacks << args.shift
+      end
+    end
+
+    # :nodoc:
+    def run_ensure_callbacks
+      @ensure_callbacks.each do | ec |
+        ec.call(self)
+      end
+    end
+
+    # :nodoc:
+    def run_ignore_callbacks
+      @ignore_callbacks.each do | ic |
+        ic.call(self)
+      end
+    end
+
+    # :nodoc:
+    def run_success_callbacks
+      @success_callbacks.each do | sc |
+        sc.call(self)
+      end
+    end
+
+    # :nodoc:
+    def run_failure_callbacks
+      @failure_callbacks.each do | fc |
+        fc.call(self)
+      end
+    end
+
+    def to_json
+      obj = {
+        :export_time => @export_time,
+        :table => @table,
+        :database => @database,
+        :state => @state,
+        :incremental_col => @incremental_col,
+        :incremental_val => @incremental_val,
+        :export_id => @export_id,
+        :filepath => @filepath,
+        :zipped => @zipped,
+        :export_type => @export_type
+      }
+      return obj.to_json
+    end
+
+    ##
+    # Final path of the dump file after zip
+    ##
+    def export_path
+      path = @zipped ? @filepath + ".gz" : @filepath
+      return path
+    end
+
+    def store!
+      cmd = "echo \"#{self.to_json.gsub("\"","\\\\\"")}\" > #{@filepath}.json"
+      puts cmd
+      result = @ssh.exec!(cmd)
+      puts result
+    end
+
+    def load metadata_path
+      json = File.open(metadata_path, "rb").read
+      hash = JSON.parse(json)
+      set_attributes hash
+    end
+
+    def set_attributes options
+      options.symbolize_keys!
+      @export_time = options[:export_time] if options[:export_time]
+      @table = options[:table] if options[:table]
+      @database = options[:database] if options[:database]
+      @state = options[:state] if options[:state] 
+      @incremental_col = options[:incremental_col] if options[:incremental_col]
+      @incremental_val = options[:incremental_val] if options[:incremental_val]
+      @export_id = options[:export_id] if options[:export_id]
+      @filepath = options[:filepath].nil? ? nil : options[:filepath]
+      @on_duplicate = options[:on_duplicate] if options[:on_duplicate]
+      @export_type = options[:export_type] if options[:export_type]
+      @zipped = options[:zipped].nil? ? false : options[:zipped]
+      @ssh = nil
+
+      @success_callbacks = []
+      @failure_callbacks = []
+      @ensure_callbacks = []
+      @ignore_callbacks = []
+    end
+
+  end
+end

data/lib/exporter/export_metadata.rb~

@@ -0,0 +1,9 @@
+module Myreplicator
+  class ExportMetadata
+    
+    def initialize *args
+      options = args.extract_options!
+    end
+    
+  end
+end

data/lib/exporter/mysql_exporter.rb

@@ -0,0 +1,187 @@
+module Myreplicator
+  class MysqlExporter
+    
+    def initialize *args
+      options = args.extract_options!
+    end
+    
+    ##
+    # Gets an Export object and dumps the data 
+    # Initially using mysqldump
+    # Incrementally using mysql -e afterwards
+    ##
+    def export_table export_obj
+      @export_obj = export_obj
+
+      ExportMetadata.record(:table => @export_obj.table_name,
+                            :database => @export_obj.source_schema,
+                            :export_id => @export_obj.id,
+                            :filepath => filepath,
+                            :incremental_col => @export_obj.incremental_column) do |metadata|
+
+        metadata.on_failure do |m|
+          update_export(:state => "failed", :export_finished_at => Time.now, :error => metadata.error)
+        end
+
+        prepare metadata
+
+        if @export_obj.state == "new"
+          initial_export metadata
+          metadata.on_success do |m|
+            metadata.state = "export_completed"
+            wrapup metadata
+          end
+        elsif !is_running?
+          # local max value for incremental export
+          max_value = incremental_export(metadata)
+
+          metadata.incremental_val = max_value # store max val in metadata
+
+          # Call back that updates the maximum value of incremental col
+          metadata.on_success do |m|
+            metadata.state = "export_completed"
+            wrapup metadata
+            @export_obj.update_max_val(max_value) #  update max value if export was successful
+          end
+        end
+        
+      end
+    end
+    
+    ##
+    # Setups SSH connection to remote host
+    ##
+    def prepare metadata
+      ssh = @export_obj.ssh_to_source
+      metadata.ssh = ssh
+    end
+    
+    ##
+    # Throws ExportIgnored if the job is still running
+    # Checks the state of the job using PID and state
+    ##
+    def is_running?
+      return false if @export_obj.state != "exporting"
+      begin
+        Process.getpgid(@export_obj.exporter_pid)
+        raise Exceptions::ExportIgnored.new("Ignored")
+      rescue Errno::ESRCH
+        return false
+      end
+    end
+
+    def update_export *args
+      options = args.extract_options!
+      @export_obj.update_attributes! options
+    end
+
+    ##
+    # File path on remote server
+    ##
+    def filepath
+      File.join(Myreplicator.configs[@export_obj.source_schema]["ssh_tmp_dir"], @export_obj.filename)
+    end
+
+    ##
+    # Exports Table using mysqldump. This method is invoked only once.
+    # Dumps with create options, no need to create table manaully
+    ##
+
+    def initial_export metadata
+      flags = ["create-options", "single-transaction"]       
+      cmd = SqlCommands.mysqldump(:db => @export_obj.source_schema,
+                                  :flags => flags,
+                                  :filepath => filepath,
+                                  :table_name => @export_obj.table_name)     
+      
+      metadata.export_type = "initial"
+
+      update_export(:state => "exporting", :export_started_at => Time.now, :exporter_pid => Process.pid)
+
+      puts "Exporting..."
+      result = execute_export(cmd, metadata)
+      check_result(result, 0)
+    end
+
+    ##
+    # Exports table incrementally, using the incremental column specified
+    # If column is not specified, it will export the entire table 
+    # Maximum value of the incremental column is recorded BEFORE export starts
+    ##
+
+    def incremental_export metadata
+      max_value = @export_obj.max_value
+      @export_obj.update_max_val if @export_obj.max_incremental_value.blank?   
+
+      sql = SqlCommands.export_sql(:db => @export_obj.source_schema,
+                                   :table => @export_obj.table_name,
+                                   :incremental_col => @export_obj.incremental_column,
+                                   :incremental_col_type => @export_obj.incremental_column_type,
+                                   :incremental_val => @export_obj.max_incremental_value)      
+
+      cmd = SqlCommands.mysql_export(:db => @export_obj.source_schema,
+                                     :filepath => filepath,
+                                     :sql => sql)
+      
+      metadata.export_type = "incremental"
+      update_export(:state => "exporting", :export_started_at => Time.now, :exporter_pid => Process.pid)
+      puts "Exporting..."
+      result = execute_export(cmd, metadata)
+      check_result(result, 0)
+    end
+
+
+    ##
+    # Completes an export process
+    # Zips files, updates states etc
+    ##
+    def wrapup metadata
+      puts "Zipping..."
+      zipfile(metadata)
+      update_export(:state => "export_completed", :export_finished_at => Time.now)
+      puts "Done.."
+    end
+
+    ##
+    # Checks the returned resut from SSH CMD
+    # Size specifies if there should be any returned results or not
+    ##
+    def check_result result, size
+      unless result.nil?
+        raise Exceptions::ExportError.new("Export Error\n#{result}") if result.length > 0
+      end     
+    end
+
+    ##
+    # Executes export command via ssh on the source DB
+    # Updates/interacts with the metadata object
+    ##
+    def execute_export cmd, metadata
+      metadata.store!
+      result = ""
+
+      # Execute Export command on the source DB server
+      result = metadata.ssh.exec!(cmd)
+      
+      return result
+    end
+
+    ##
+    # zips the file on the source DB server
+    ##
+    def zipfile metadata
+      cmd = "cd #{Myreplicator.configs[@export_obj.source_schema]["ssh_tmp_dir"]}; gzip #{@export_obj.filename}"
+      
+      zip_result = metadata.ssh.exec!(cmd)
+
+      unless zip_result.nil?        
+        raise Exceptions::ExportError.new("Export Error\n#{zip_result}") if zip_result.length > 0
+      end
+
+      metadata.zipped = true
+
+      return zip_result
+    end
+    
+  end
+end