myreplicator 0.0.1

data/lib/exporter/sql_commands.rb

@@ -0,0 +1,126 @@
+module Myreplicator
+  module SqlCommands
+    
+    def self.mysqldump *args
+      options = args.extract_options!
+      options.reverse_merge! :flags => []
+      db = options[:db]
+
+      flags = ""
+
+      self.dump_flags.each_pair do |flag, value|
+        if options[:flags].include? flag
+          flags += " --#{flag} "
+        elsif value
+          flags += " --#{flag} "
+        end
+      end
+
+      # Database host when ssh'ed into the db server
+      db_host = ssh_configs(db)["ssh_db_host"].nil? ? "127.0.0.1" : ssh_configs(db)["ssh_db_host"]
+
+      cmd = Myreplicator.mysqldump
+      cmd += "#{flags} -u#{db_configs(db)["username"]} -p#{db_configs(db)["password"]} "
+      cmd += "-h#{db_host} " if db_configs(db)["host"]
+      cmd += " -P#{db_configs(db)["port"]} " if db_configs(db)["port"]
+      cmd += " #{db} "
+      cmd += " #{options[:table_name]} "
+      cmd += "--result-file=#{options[:filepath]} "
+
+      # cmd += "--tab=#{options[:filepath]} "
+      # cmd += "--fields-enclosed-by=\'\"\' "
+      # cmd += "--fields-escaped-by=\'\\\\\' "
+
+      return cmd
+    end
+
+    def self.db_configs db
+      ActiveRecord::Base.configurations[db]
+    end
+
+    def self.ssh_configs db
+      Myreplicator.configs[db]
+    end
+
+    def self.dump_flags
+      {"add-locks" => true,
+        "compact" => false,
+        "lock-tables" => false,
+        "no-create-db" => true,
+        "no-data" => false,
+        "quick" => true,
+        "skip-add-drop-table" => false,
+        "create-options" => false,
+        "single-transaction" => false
+      }
+    end
+
+    def self.mysql_export *args
+      options = args.extract_options!
+      options.reverse_merge! :flags => []
+      db = options[:db]
+      # Database host when ssh'ed into the db server
+      db_host = ssh_configs(db)["ssh_db_host"].nil? ? "127.0.0.1" : ssh_configs(db)["ssh_db_host"]
+
+      flags = ""
+
+      self.mysql_flags.each_pair do |flag, value|
+        if options[:flags].include? flag
+          flags += " --#{flag} "
+        elsif value
+          flags += " --#{flag} "
+        end
+      end
+
+      cmd = Myreplicator.mysql
+      cmd += "#{flags} -u#{db_configs(db)["username"]} -p#{db_configs(db)["password"]} " 
+      cmd += "-h#{db_host} " if db_configs(db)["host"].blank?
+      cmd += db_configs(db)["port"].blank? ? "-P3306 " : "-P#{db_configs(db)["port"]} "
+      cmd += "--execute=\"#{options[:sql]}\" "
+      cmd += " > #{options[:filepath]} "
+      
+      puts cmd
+      return cmd
+    end
+
+    def self.mysql_flags
+      {"column-names" => false,
+        "quick" => true,
+        "reconnect" => true
+      }    
+    end
+
+    def self.export_sql *args
+      options = args.extract_options!
+      sql = "SELECT * FROM #{options[:db]}.#{options[:table]} " 
+      
+      if options[:incremental_col] && options[:incremental_val]
+        if options[:incremental_col_type] == "datetime"
+          sql += "WHERE #{options[:incremental_col]} >= '#{options[:incremental_val]}'"
+        else
+          sql += "WHERE #{options[:incremental_col]} >= #{options[:incremental_val]}"
+        end
+      end
+
+      return sql
+    end
+
+    def self.max_value_sql *args
+      options = args.extract_options!
+      sql = ""
+
+      if options[:incremental_col]
+        sql = "SELECT max(#{options[:incremental_col]}) FROM #{options[:db]}.#{options[:table]}" 
+      else
+        raise Myreplicator::Exceptions::MissingArgs.new("Missing Incremental Column Parameter")
+      end
+      
+      return sql
+    end
+
+    def self.mysql_export_outfile
+      
+    end
+
+  end
+end

data/lib/exporter/sql_commands.rb~

@@ -0,0 +1,5 @@
+module Myreplicator
+  module SqlComands
+    
+  end
+end

data/lib/exporter.rb

@@ -0,0 +1,3 @@
+Dir["#{File.expand_path(File.dirname(__FILE__))}/exporter/*.rb"].each { | f | require File.expand_path(f) }
+Dir["#{File.expand_path(File.dirname(__FILE__))}/transporter/*.rb"].each { | f | require File.expand_path(f) }
+Dir["#{File.expand_path(File.dirname(__FILE__))}/loader/*.rb"].each { | f | require File.expand_path(f) }

data/lib/loader/import_sql.rb

@@ -0,0 +1,91 @@
+module Myreplicator
+  class ImportSql
+    class << self
+      
+      def load_data_infile *args
+        options = args.extract_options!
+        sql = build_load_data_infile options
+
+        cmd = mysql_cmd(options[:db])
+        cmd += " --local_infile=1 " # Used for mysql versions that do not support -e and LDI
+        cmd += "-e \"#{sql}\" "
+
+        return cmd
+      end
+      
+      def build_load_data_infile options
+        options.reverse_merge!(:replace => true,
+                               :fields_terminated_by => "\\t",
+                               :lines_terminated_by => "\\n"
+                               )
+                               
+
+        handle = options[:replace] ? 'REPLACE' : 'IGNORE' 
+        
+        sql =  "LOAD DATA LOCAL INFILE '#{options[:filepath]}' #{handle} "
+        sql += "INTO TABLE #{options[:db]}.#{options[:table_name]} "
+        
+        if options.include?(:character_set)
+          sql << " CHARACTER SET #{options[:character_set]}"
+        end
+        
+        if options.include?(:fields_terminated_by) or options.include?(:enclosed_by) or options.include?(:escaped_by)
+          sql << " FIELDS"
+        end
+        if options.include?(:fields_terminated_by)
+          sql << " TERMINATED BY '#{options[:fields_terminated_by]}'"
+        end
+        if options.include?(:enclosed_by)
+          sql << " ENCLOSED BY '#{options[:enclosed_by]}'"
+        end
+        if options.include?(:escaped_by)
+          sql << " ESCAPED BY '#{options[:escaped_by]}'"
+        end
+        
+        if options.include?(:starting_by) or options.include?(:lines_terminated_by)
+          sql << " LINES"
+        end
+        if options.include?(:starting_by) 
+          sql << " STARTING BY '#{options[:starting_by]}'"
+        end
+        if options.include?(:lines_terminated_by)
+          sql << " TERMINATED BY '#{options[:lines_terminated_by]}'"
+        end
+
+        if options.include?(:ignore)
+          sql << " IGNORE #{options[:ignore]} LINES"
+        end
+        
+        if options.include?(:fields) and !options[:fields].empty?
+          sql << "( #{options[:fields].join(', ')} )"
+        end
+
+        return sql
+      end
+
+      def initial_load *args
+        options = args.extract_options!
+
+        cmd = mysql_cmd(options[:db])       
+        cmd += " #{options[:db]} "
+        cmd += " < #{options[:filepath]} "
+        
+        return cmd
+      end
+      
+      def mysql_cmd db
+        # Destination database host
+        db_host = SqlCommands.db_configs(db).has_key?("host") ? SqlCommands.db_configs(db)["host"] : "127.0.0.1"
+        
+        cmd = Myreplicator.mysql
+        cmd += " -u#{SqlCommands.db_configs(db)["username"]} -p#{SqlCommands.db_configs(db)["password"]} "
+        cmd += " -h#{db_host} " 
+        cmd += " -P#{SqlCommands.db_configs(db)["port"]} " if SqlCommands.db_configs(db)["port"]
+        
+        return cmd
+      end
+      
+    end #self
+
+  end
+end

data/lib/loader/import_sql.rb~

@@ -0,0 +1,27 @@
+module Myreplicator
+  module ImportSql
+
+    def self.load_data_infile *args
+      options = args.extract_options!
+      sql = ""
+    end
+
+    def self.initial_load *args
+      options = args.extract_options!
+      db = options[:db]
+      cmd = ""
+
+      # Destination database host
+      db_host = db_configs(db).has_key?("host") ? db_configs(db)["host"] : "127.0.0.1"
+      
+      cmd = Myreplicator.mysql
+      cmd += "-u#{db_configs(db)["username"]} -p#{db_configs(db)["password"]} "
+      cmd += "-h#{db_host} " 
+      cmd += " -P#{db_configs(db)["port"]} " if db_configs(db)["port"]
+      cmd += " #{db} "
+      cmd += " #{options[:table_name]} "
+      cmd += " < #{options[:filepath]} "
+    end
+
+  end
+end

data/lib/loader/loader.rb

@@ -0,0 +1,122 @@
+require "exporter"
+
+module Myreplicator
+  class Loader
+    
+    def initialize *args
+      options = args.extract_options!
+    end
+    
+    def tmp_dir
+      @tmp_dir ||= File.join(Myreplicator.app_root,"tmp", "myreplicator")
+    end
+
+    def load
+      initials = []
+      incrementals = []
+
+      metadata_files.each do |metadata|
+        if metadata.export_type == "initial"
+          initials << metadata
+        else
+          incrementals << metadata
+        end
+      end
+      
+      initials.each do |metadata| 
+        puts metadata.table
+        initial_load metadata
+        cleanup metadata
+      end
+
+      incrementals.each do |metadata|
+        puts metadata.table
+        incremental_load metadata
+        cleanup metadata
+      end
+    end
+
+    ##
+    # Creates table and loads data
+    ##
+    def initial_load metadata
+      exp = Export.find(metadata.export_id)
+      unzip(metadata.filename)
+      metadata.zipped = false
+
+      cmd = ImportSql.initial_load(:db => exp.destination_schema,
+                                   :filepath => metadata.destination_filepath(tmp_dir))      
+      puts cmd
+      result = `#{cmd}` # execute
+      unless result.nil?
+        if result.size > 0
+          raise Exceptions::LoaderError.new("Initial Load #{metadata.filename} Failed!\n#{result}") 
+        end
+      end
+    end
+
+    ##
+    # Loads data incrementally
+    # Uses the values specified in the metadatta object
+    ##
+    def incremental_load metadata
+      exp = Export.find(metadata.export_id)
+      unzip(metadata.filename)
+      metadata.zipped = false
+
+      cmd = ImportSql.load_data_infile(:table_name => exp.table_name, 
+                                       :db => exp.destination_schema,
+                                       :filepath => metadata.destination_filepath(tmp_dir))
+      puts cmd
+      result = `#{cmd}` # execute
+      unless result.nil?
+        if result.size > 0
+          raise Exceptions::LoaderError.new("Incremental Load #{metadata.filename} Failed!\n#{result}") 
+        end
+      end
+    end
+
+    ##
+    # Deletes the metadata file and extract
+    ##
+    def cleanup metadata
+      puts "Cleaning up..."
+      FileUtils.rm "#{metadata.destination_filepath(tmp_dir)}.json" # json file
+      FileUtils.rm metadata.destination_filepath(tmp_dir) # dump file
+    end
+
+    ##
+    # Unzips file
+    # Checks if the file exists or already unzipped
+    ##
+    def unzip filename
+      cmd = "cd #{tmp_dir}; gunzip #{filename}"
+      passed = false
+      if File.exist?(File.join(tmp_dir,filename))
+        result = `#{cmd}`
+        unless result.nil? 
+          puts result
+          unless result.length > 0
+            passed = true
+          end
+        else
+          passed = true
+        end
+      elsif File.exist?(File.join(tmp_dir,filename.gsub(".gz","")))
+        puts "File already unzipped"
+        passed = true
+      end
+
+      raise Exceptions::LoaderError.new("Unzipping #{filename} Failed!") unless passed
+    end
+
+    def metadata_files
+      files = []
+      Dir.glob(File.join(tmp_dir, "*.json")).each do |json_file|
+        files << ExportMetadata.new(:metadata_path => json_file)
+      end
+      return files
+    end
+
+  end
+end

data/lib/loader/loader.rb~

@@ -0,0 +1,4 @@
+module Myreplicator
+  class Transporter
+  end
+end

data/lib/myreplicator/engine.rb

@@ -0,0 +1,5 @@
+module Myreplicator
+  class Engine < ::Rails::Engine
+    isolate_namespace Myreplicator
+  end
+end

data/lib/myreplicator/version.rb

@@ -0,0 +1,3 @@
+module Myreplicator
+  VERSION = "0.0.1"
+end

data/lib/myreplicator.rb

@@ -0,0 +1,34 @@
+require "myreplicator/engine"
+
+
+module Myreplicator
+  mattr_accessor :app_root, :tmp_path, :mysql, :mysqldump, :configs
+  
+  class Engine < Rails::Engine
+
+    initializer "myreplicator.configure_rails_initialization" do |app|
+      Myreplicator.app_root = app.root #assigning app root as rails.root is not accessible
+      yml = YAML.load(File.read("#{Myreplicator.app_root}/config/myreplicator.yml"))
+
+      Myreplicator.mysql = yml["myreplicator"]["mysql"]
+      Myreplicator.mysqldump = yml["myreplicator"]["mysqldump"]
+      
+      if yml["myreplicator"]["tmp_path"].blank?
+        Myreplicator.tmp_path = File.join(Myreplicator.app_root, "tmp", "myreplicator")
+      else
+        Myreplicator.tmp_path = yml["myreplicator"]["tmp_path"]
+      end
+      
+      Myreplicator.configs = yml
+    end
+
+  end
+
+  module Exceptions
+    class MissingArgs < StandardError; end
+    class ExportError < StandardError; end
+    class LoaderError < StandardError; end
+    class ExportIgnored < StandardError; end
+  end
+
+end

data/lib/tasks/myreplicator_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :myreplicator do
+#   # Task goes here
+# end

data/lib/transporter/parallelizer.rb

@@ -0,0 +1,78 @@
+require "thread"
+
+module Myreplicator
+  class Parallelizer
+
+    attr_accessor :queue
+
+    def initialize *args
+      options = args.extract_options!
+      @queue = Queue.new
+      @threads = []
+      @max_threads = options[:max_threads].nil? ? 10 : options[:max_threads]     
+    end
+
+    ##
+    # Runs while there are jobs in the queue
+    # Waits for a second and checks for available threads
+    # Exits when all jobs are allocated in threads
+    ##
+    def run
+      @done = false
+      @manager_running = false
+
+      while @queue.size > 0
+        if @threads.size <= @max_threads
+          @threads << Thread.new(@queue.pop) do |proc|
+            Thread.current[:status] = 'running' # Manually Set Thread state for Checks
+            Transporter.new.instance_exec(proc[:params], &proc[:block])
+            Thread.current[:status] = 'done' 
+          end
+        else
+          unless @manager_running
+            manage_threads 
+            @manager_running = true
+          end
+          sleep 1
+        end
+      end   
+
+      manage_threads unless @manager_running
+
+      # Waits until all threads are completed
+      # Before exiting
+      while !@done
+        sleep 1
+      end
+
+    end
+    
+    ##
+    # Clears dead threads, 
+    # frees thread pool for more jobs
+    # Exits when no more threads are left
+    ##
+    def manage_threads
+      Thread.new do 
+        while(@threads.size > 0)
+          done = []
+          @threads.each do |t|
+            done << t if t[:status] == "done"
+          end
+
+          done.each{|d| @threads.delete(d)} # Clear dead threads
+          
+          # If no more jobs are left, mark done
+
+          if @queue.size == 0 && @threads.size == 0
+            @done = true
+          else
+            sleep 2 # Wait for more threads to spawn
+          end
+
+        end
+      end
+    end
+
+  end
+end

data/lib/transporter/transporter.rb

@@ -0,0 +1,80 @@
+require "exporter"
+
+module Myreplicator
+  class Transporter
+    
+    def initialize *args
+      options = args.extract_options!
+    end
+
+    def tmp_dir
+      @tmp_dir ||= File.join(Myreplicator.app_root,"tmp", "myreplicator")
+      Dir.mkdir(@tmp_dir) unless File.directory?(@tmp_dir)
+      @tmp_dir
+    end
+
+    def transfer
+      unique_jobs = Export.where("state != 'failed' and active = 1").group("source_schema")
+      
+      unique_jobs.each do |export|
+        download export
+      end
+    end
+
+    def download export
+      ssh = export.ssh_to_source     
+      parallel_download(export, ssh, completed_files(ssh, export))
+    end
+    
+    def parallel_download export, ssh, files    
+      p = Parallelizer.new
+      
+      files.each do |filename|
+        puts filename
+        p.queue << {:params =>[ssh, export, filename], :block => download_file}
+      end
+
+      p.run 
+    end
+
+    def download_file    
+      proc = Proc.new { |params|
+        ssh = params[0]
+        export = params[1] 
+        filename = params[2]
+        sftp = export.sftp_to_source
+        json_file = remote_path(export, filename) 
+        json_local_path = File.join(tmp_dir,filename)
+        puts "Downloading #{json_file}"
+        sftp.download!(json_file, json_local_path)
+        dump_file = get_dump_path(json_local_path)
+        puts "Downloading #{dump_file}"
+        sftp.download!(dump_file, File.join(tmp_dir, dump_file.split("/").last))             
+      }
+    end
+
+    def completed_files ssh, export
+      done_files = ssh.exec!(get_done_files(export))
+
+      unless done_files.blank?
+        return done_files.split("\n")
+      end
+
+      return []
+    end
+
+    def get_dump_path json_path
+      metadata = ExportMetadata.new(:metadata_path => json_path)
+      return metadata.export_path
+    end
+
+    def remote_path export, filename
+      File.join(Myreplicator.configs[export.source_schema]["ssh_tmp_dir"], filename)
+    end
+
+    def get_done_files export
+      cmd = "cd #{Myreplicator.configs[export.source_schema]["ssh_tmp_dir"]}; grep -l export_completed *.json"
+    end
+    
+  end
+end