myreplicator 0.0.16 → 0.0.17

data/app/assets/stylesheets/myreplicator/application.css

@@ -129,7 +129,7 @@ html { overflow-y:scroll;overflow-x:auto; }
   padding-left:20px;
   margin-top:-10px;
 }
-.home-menu span {
+.home-menu > span {
   color:#474747;
   display:block;
   float:left;
@@ -150,6 +150,7 @@ html { overflow-y:scroll;overflow-x:auto; }
   letter-spacing:1px;
   margin:8px 15px 8px 0; 
   padding:3px 12px;
+  position:relative;
   text-decoration:none;
   -webkit-transition: all 0.25s ease-in-out;
   -moz-transition: all 0.25s ease-in-out;
@@ -178,7 +179,31 @@ html { overflow-y:scroll;overflow-x:auto; }
   text-shadow: 0 1px #f9f9f9;  
 }
 .home-menu a:active {background:#e0e0e0;}
-
+.home-menu a span {
+  background:#990000;
+  box-shadow:0 0 3px rgba(0,0,0,0.4);
+  border-radius:999px;
+  color:#fff;
+  font-size:11px;
+  font-weight:normal;
+  height:13px;
+  display:block;
+  opacity:0;
+  padding:0 0 3px 1px;
+  position:absolute;
+  top:-7px;
+  right:-5px;
+  text-align:center;
+  text-shadow:1px 1px #333;
+  width:13px;
+  -webkit-transition: all 0.25s ease-in-out;
+  -moz-transition: all 0.25s ease-in-out;
+  -o-transition: all 0.25s ease-in-out;
+  transition: all 0.25s ease-in-out;     
+}
+.home-menu a.overview span {background:#1F6015;}
+.home-menu a:hover span,
+.home-menu a.on span {opacity:1;}
 #sub-header a.active{
   background:#666;  
   background-image: -moz-linear-gradient(top, #555, #777);
@@ -203,6 +228,14 @@ h2 {
 	margin:20px;
 }
 
+h6 {
+  color:#474747;
+  font-size:14px;
+  font-weight:normal;
+  text-shadow:0 1px #fff;
+  margin:10px 5px;
+}
+
 form {margin:5px 20px;}
 form div.form-section {
   width:400px;
@@ -324,15 +357,9 @@ div.flash {
   z-index:110;
 }
 
-div.table-wrapper {
-  margin:10px 20px;
-}
+div.table-wrapper { margin:10px 20px; }
 
-table.data-grid {
-  font-size:12px; 
-  margin:10px 0;
-  width:100%;
-}
+table.data-grid { font-size:12px; margin:10px 0; width:100%; }
 table.data-grid thead tr,
 #states-list thead tr{
   background-color:#ddd;
@@ -542,3 +569,19 @@ p.hint {color:#474747;font-size:12px;line-height:20px;margin:8px 5px;width:300px
   margin:0 0 10px;  
   text-transform:capitalize;
 }
+
+table.overview {
+  width:100%;
+}
+
+table.overview td {
+  border-bottom:1px solid #eee;
+  font-size:12px;
+  padding:5px 8px 5px 5px;
+  text-shadow: 0 1px #fff;
+  vertical-align:middle;
+}
+table.overview tr:nth-child(even) {background:#fbfbfb;}
+table.overview tr:last-child td {border-bottom:0px;}
+table.overview td span.name {color:#474747;display:block;font-weight:bold; font-size:13px;padding-bottom:5px;}
+table.overview td span.file {color:#999;display:block;font-style:italic;}

data/app/controllers/myreplicator/exports_controller.rb

@@ -31,7 +31,8 @@ module Myreplicator
     # GET /exports/new.json
     def new
       @export = Export.new
-  
+      @dbs = get_dbs
+      @tables = db_metadata
       respond_to do |format|
         format.html # new.html.erb
         format.json { render json: @export }
@@ -41,6 +42,8 @@ module Myreplicator
     # GET /exports/1/edit
     def edit
       @export = Export.find(params[:id])
+      @dbs = get_dbs
+      @tables = db_metadata
       @edit = true
     end
   
@@ -48,6 +51,7 @@ module Myreplicator
     # POST /exports.json
     def create
       @export = Export.new(params[:export])
+      @dbs = get_dbs
   
       respond_to do |format|
         if @export.save
@@ -64,6 +68,7 @@ module Myreplicator
     # PUT /exports/1.json
     def update
       @export = Export.find(params[:id])
+      @dbs = get_dbs
   
       respond_to do |format|
         if @export.update_attributes(params[:export])
@@ -101,6 +106,17 @@ module Myreplicator
     def sort_direction
       %w[asc desc].include?(params[:direction]) ? params[:direction] : "asc"
     end
+    
+    def db_metadata
+      @db_metadata ||= Myreplicator::Export.available_tables
+    end
 
+    def get_dbs
+      return db_metadata.keys
+    end
+
+    def get_tables(db)
+      return db_metadata[db]
+    end
   end
 end

data/app/controllers/myreplicator/home_controller.rb

@@ -7,6 +7,8 @@ module Myreplicator
       @tab = 'home'
       @option = 'overview'
       @exports = Export.order('state DESC')
+      @logs = Log.where(:state => 'running').order("started_at DESC")
+      @now = Time.zone.now
       respond_to do |format|
         format.html # index.html.erb
         format.json { render json: @exports }
@@ -16,7 +18,8 @@ module Myreplicator
     def errors
       @tab = 'home'
       @option = 'errors'
-      @exports = Export.where("error is not null").order('source_schema ASC')     
+      @exports = Export.where("error is not null").order('source_schema ASC')    
+      @logs = Log.where(:state => 'error').order("started_at DESC") 
     end
     
   end

data/app/helpers/myreplicator/application_helper.rb

@@ -8,5 +8,25 @@ module Myreplicator
       link_to content_tag(:span, title), {:sort => column, :direction => direction}, {:class => css_class}
     end
 
+    def chronos(secs)
+      [[60, :seconds], [60, :minutes], [24, :hours], [1000, :days]].map{ |count, name|
+        if secs > 0
+          secs, n = secs.divmod(count)
+          "#{n.to_i} #{name}"
+        end
+      }.compact.reverse.join(', ')
+    end
+
+    def err_count
+      total = Log.where(:state => 'error').count
+      return total
+    end
+
+
+    def run_count
+      total = Log.where(:state => 'running').count
+      return total
+    end
+
   end
 end

data/app/models/myreplicator/export.rb

@@ -28,8 +28,22 @@ module Myreplicator
                     )
 
     attr_reader :filename
-    
-    def export
+
+    @queue = :myreplicator_export # Provided for Resque
+
+    ##
+    # Perfoms the export job, Provided for Resque
+    ##
+    def self.perform(export_id, *args)
+      options = args.extract_options!
+      export_obj = Export.find(export_id)
+      export_obj.export_table
+    end
+
+    ##
+    # Runs the export process using the required Exporter library
+    ##
+    def export_table
       exporter = MysqlExporter.new
       exporter.export_table self    
     end
@@ -70,34 +84,90 @@ module Myreplicator
       puts "Connecting SFTP..."
       return connection_factory(:sftp)
     end
-
+    
+    ##
+    # Connects to the server via ssh/sftp
+    ##
     def connection_factory type
+      config = Myreplicator.configs[self.source_schema]
+
       case type
       when :ssh
-        if Myreplicator.configs[self.source_schema].has_key? "ssh_password"
-          return Net::SSH.start(Myreplicator.configs[self.source_schema]["ssh_host"],
-                                Myreplicator.configs[self.source_schema]["ssh_user"],
-                                :password => Myreplicator.configs[self.source_schema]["ssh_password"])
-
-        elsif(Myreplicator.configs[self.source_schema].has_key? "ssh_private_key")
-          return Net::SSH.start(Myreplicator.configs[self.source_schema]["ssh_host"],
-                                Myreplicator.configs[self.source_schema]["ssh_user"],
-                                :keys => [Myreplicator.configs[self.source_schema]["ssh_private_key"]])        
+        if config.has_key? "ssh_password"
+          return Net::SSH.start(config["ssh_host"], config["ssh_user"], :password => config["ssh_password"])
+
+        elsif(config.has_key? "ssh_private_key")
+          return Net::SSH.start(config["ssh_host"], config["ssh_user"], :keys => [config["ssh_private_key"]])
         end
       when :sftp
-        if Myreplicator.configs[self.source_schema].has_key? "ssh_password"
-          return Net::SFTP.start(Myreplicator.configs[self.source_schema]["ssh_host"],
-                                 Myreplicator.configs[self.source_schema]["ssh_user"],
-                                 :password => Myreplicator.configs[self.source_schema]["ssh_password"])
-
-        elsif(Myreplicator.configs[self.source_schema].has_key? "ssh_private_key")
-          return Net::SFTP.start(Myreplicator.configs[self.source_schema]["ssh_host"],
-                                 Myreplicator.configs[self.source_schema]["ssh_user"],
-                                 :keys => [Myreplicator.configs[self.source_schema]["ssh_private_key"]])
+        if config.has_key? "ssh_password"
+          return Net::SFTP.start(config["ssh_host"], config["ssh_user"], :password => config["ssh_password"])
+
+        elsif(config.has_key? "ssh_private_key")
+          return Net::SFTP.start(config["ssh_host"], config["ssh_user"], :keys => [config["ssh_private_key"]])
         end          
       end
     end
 
+    ##
+    # Returns a hash of {DB_NAME => [TableName1,...], DB => ...}
+    ##
+    def self.available_tables
+      metadata = {}
+      available_dbs.each do |db|
+        tables = SourceDb.get_tables(db)
+        metadata[db] = tables
+      end
+      return metadata
+    end
+
+    ##
+    # List of all avaiable databases from database.yml file
+    # All Export/Load jobs can use these databases
+    ##
+    def self.available_dbs
+      dbs = ActiveRecord::Base.configurations.keys
+      dbs.delete("development")
+      dbs.delete("test")
+      return dbs
+    end
+
+    ##
+    # NOTE: Provided for Resque use
+    # Schedules all the exports in resque
+    # Requires Resque Scheduler
+    ##
+    def self.schedule_in_resque
+      exports = Export.find(:all)
+      exports.each do |export|
+        if export.active
+          export.schedule
+        else
+          Resque.remove_schedule(export.schedule_name)
+        end
+      end
+      Resque.reload_schedule! # Reload all schedules in Resque
+    end
+
+    ##
+    # Name used for the job in Resque
+    ##
+    def schedule_name
+      name = "#{source_schema}_#{destination_schema}_#{table_name}"
+    end
+
+    ##
+    # Schedules the export job in Resque
+    ##
+    def schedule
+      Resque.set_schedule(schedule_name, {
+                            :cron => cron,
+                            :class => "Myreplicator::Export",
+                            :queue => "myreplicator_export",
+                            :args => id
+                          })
+    end
+
     ##
     # Inner Class that connects to the source database 
     # Handles connecting to multiple databases
@@ -106,9 +176,24 @@ module Myreplicator
     class SourceDb < ActiveRecord::Base
       
       def self.connect db
-        @@connected ||= true
         establish_connection(ActiveRecord::Base.configurations[db])
-        Kernel.p ActiveRecord::Base.connected?
+      end
+
+      ##
+      # Returns tables as an Array
+      # releases the connection
+      ##
+      def self.get_tables(db)
+        tables = []
+        begin
+          self.connect(db)
+          tables = self.connection.tables  
+          self.connection_pool.release_connection
+        rescue Mysql2::Error => e
+          puts "Connection to #{db} Failed!"
+          puts e.message
+        end          
+        return tables
       end
       
       def self.exec_sql source_db,sql

data/app/models/myreplicator/log.rb

@@ -0,0 +1,90 @@
+module Myreplicator
+  class Log < ActiveRecord::Base
+    attr_accessible(:pid,
+                    :job_type,
+                    :name,
+                    :file,
+                    :state,
+                    :thread_state,
+                    :hostname,
+                    :export_id,
+                    :error,
+                    :backtrace,
+                    :guid,
+                    :started_at,
+                    :finished_at)
+    
+    ##
+    # Creates a log object
+    # Stores the state and related information about the job
+    # File's names are supposed to be unique
+    ##
+    def self.run *args
+      options = args.extract_options!
+      options.reverse_merge!(:started_at => Time.now,
+                             :pid => Process.pid,
+                             :hostname => Socket.gethostname,
+                             :guid => SecureRandom.hex(5),
+                             :thread_state => Thread.current.status,
+                             :state => "new")
+
+      log = Log.create options
+
+      unless log.running?
+        begin
+          log.state = "running"
+          log.save!
+
+          yield log
+
+          log.state = "completed"
+        rescue Exception => e
+          log.state = "error"
+          log.error = e.message
+          log.backtrace =  e.backtrace
+
+        ensure
+          log.finished_at = Time.now
+          log.thread_state = Thread.current.status
+          log.save!
+        end
+      end
+
+    end
+
+    ##
+    # Kills the job if running
+    # Using PID
+    ##
+    def kill
+      begin
+        Process.kill('TERM', pid)
+      rescue Errno::ESRCH
+        puts "pid #{pid} does not exist!"
+      end
+    end
+
+    ##
+    # Checks to see if the PID of the log is active or not
+    ##
+    def running?
+      logs = Log.where(:file => file, :job_type => job_type, :state => "running")
+  
+      if logs.count > 0
+        logs.each do |log|
+          begin
+            Process.getpgid(log.pid)
+            puts "still running #{filepath}"
+            return true
+          rescue Errno::ESRCH
+            log.state = "error"
+            log.save!
+          end
+        end
+      end
+      
+      return false
+    end
+
+  end
+end