myreplicator 1.1.20 → 1.1.21

data/app/models/myreplicator/log.rb

@@ -148,10 +148,11 @@ module Myreplicator
       log = Log.where(:export_id => options[:export_id],
                       :file => options[:file],
                       :job_type => options[:job_type]).last
-      Kernel.p "===== transport_complete? log ====="
-      Kernel.p log
+      #Kernel.p "===== transport_complete? log ====="
+      #Kernel.p log
       if log.nil?
-        return false
+        #return false
+        return true
       else
         return true if log.state == "completed"
       end

data/lib/exporter/mysql_exporter.rb

@@ -134,8 +134,8 @@ module Myreplicator
           options[:incremental_val] = @export_obj.max_incremental_value
         end
 
-        Kernel.p "===== incremental_export_into_outfile OPTIONS ====="
-        Kernel.p options
+        #Kernel.p "===== incremental_export_into_outfile OPTIONS ====="
+        #Kernel.p options
         cmd = SqlCommands.mysql_export_outfile(options)      
         exporting_state_trans
         puts "Exporting..."
@@ -152,8 +152,8 @@ module Myreplicator
     end
 
     def self.compare_schemas vertica_schema, mysql_schema
-      Kernel.p vertica_schema
-      Kernel.p mysql_schema
+      #Kernel.p vertica_schema
+      #Kernel.p mysql_schema
       if vertica_schema.size != mysql_schema.size
         return true
       else
@@ -196,8 +196,8 @@ module Myreplicator
     end
 
     def self.schema_changed? options
-      Kernel.p "===== schema_changed? ====="
-      puts options
+      #Kernel.p "===== schema_changed? ====="
+      #puts options
       mysql_schema = Loader.mysql_table_definition(options)
       vertica_schema = VerticaLoader.destination_table_vertica(options)
 
@@ -205,19 +205,16 @@ module Myreplicator
       unless vertica_schema.size > 0 
         return {:changed => true, :mysql_schema => mysql_schema, :new => true}
       end
-      Kernel.p "***** 1 *****"
       # compare two schemas
       
       
       mysql_schema_2 = get_mysql_schema_rows mysql_schema
       if compare_schemas(vertica_schema, mysql_schema_2)
-        Kernel.p "***** 2 *****"
         result =  {:changed => true, :mysql_schema => mysql_schema, :vertica_schema => vertica_schema,:new => false}
       else
-        Kernel.p "***** 3 *****"
         result =  {:changed => false, :mysql_schema => mysql_schema}
       end
-      Kernel.p result
+      #Kernel.p result
       return result
     end
 

data/lib/exporter/sql_commands.rb

@@ -145,17 +145,17 @@ module Myreplicator
     # Location of the output file needs to have 777 perms
     ##
     def self.mysql_export_outfile *args
-      Kernel.p "===== mysql_export_outfile OPTIONS ====="
+      #Kernel.p "===== mysql_export_outfile OPTIONS ====="
        
       options = args.extract_options!
-      Kernel.p options
+      #Kernel.p options
       options.reverse_merge! :flags => []
       db = options[:source_schema]
 
       # Database host when ssh'ed into the db server
       db_host = "127.0.0.1"
-      Kernel.p "===== mysql_export_outfile ssh_configs ====="
-      Kernel.p ssh_configs(db)
+      #Kernel.p "===== mysql_export_outfile ssh_configs ====="
+      #Kernel.p ssh_configs(db)
       if !ssh_configs(db)["ssh_db_host"].blank?
         db_host =  ssh_configs(db)["ssh_db_host"]
       elsif !db_configs(db)["host"].blank?

data/lib/loader/loader.rb

@@ -33,11 +33,11 @@ module Myreplicator
       incrementals = []
       all_files = Myreplicator::Loader.metadata_files
       
-      Kernel.p "===== all_files ====="
-      Kernel.p all_files 
+      #Kernel.p "===== all_files ====="
+      #Kernel.p all_files 
       
       all_files.each do |m|
-        Kernel.p m
+        #Kernel.p m
         if m.export_type == "initial"
           initials << m # Add initial to the list
           all_files.delete(m) # Delete obj from mixed list
@@ -53,11 +53,42 @@ module Myreplicator
       
       incrementals = all_files # Remaining are all incrementals
       
-      initial_procs = Loader.initial_loads initials
-      parallel_load initial_procs
-
-      incremental_procs = Loader.incremental_loads incrementals
-      parallel_load incremental_procs
+      #initial_procs = Loader.initial_loads initials
+      #parallel_load initial_procs
+      initials.each do |metadata|
+        Myreplicator::Log.run(:job_type => "loader", 
+        :name => "#{metadata.export_type}_import",
+        :file => metadata.filename,
+        :export_id => metadata.export_id) do |log|
+          if Myreplicator::Loader.transfer_completed? metadata
+            if metadata.export_to == "vertica"
+              Myreplicator::Loader.incremental_load metadata
+            else
+              Myreplicator::Loader.initial_load metadata
+            end
+            Myreplicator::Loader.cleanup metadata
+          end
+        end
+      end
+      
+      #incremental_procs = Loader.incremental_loads incrementals
+      #parallel_load incremental_procs
+      #groups = Myreplicator::Loader.group_incrementals incrementals
+      #groups.each do |group|
+      incrementals.each do |metadata|
+        Myreplicator::Log.run(:job_type => "loader",
+        :name => "incremental_import",
+        :file => metadata.filename,
+        :export_id => metadata.export_id) do |log|
+          if Myreplicator::Loader.transfer_completed? metadata
+            Myreplicator::Loader.incremental_load metadata
+            Myreplicator::Loader.cleanup metadata
+          end
+        end
+      end
+       #   end # group
+      #end # groups
+         
     end
     
     def self.parallel_load procs
@@ -78,18 +109,18 @@ module Myreplicator
 
       initials.each do |metadata| 
         procs << Proc.new {
-          Log.run(:job_type => "loader", 
+          Myreplicator::Log.run(:job_type => "loader", 
                   :name => "#{metadata.export_type}_import", 
                   :file => metadata.filename, 
                   :export_id => metadata.export_id) do |log|
 
-            if Loader.transfer_completed? metadata
+            if Myreplicator::Loader.transfer_completed? metadata
               if metadata.export_to == "vertica"
-                Loader.incremental_load metadata
+                Myreplicator::Loader.incremental_load metadata
               else
-                Loader.initial_load metadata
+                Myreplicator::Loader.initial_load metadata
               end
-              Loader.cleanup metadata
+              Myreplicator::Loader.cleanup metadata
             end
 
           end
@@ -110,14 +141,14 @@ module Myreplicator
       groups.each do |group|
         procs << Proc.new {
           group.each do |metadata|
-            Log.run(:job_type => "loader", 
+            Myreplicator::Log.run(:job_type => "loader", 
                     :name => "incremental_import", 
                     :file => metadata.filename, 
                     :export_id => metadata.export_id) do |log|
     
-              if Loader.transfer_completed? metadata            
-                Loader.incremental_load metadata
-                Loader.cleanup metadata
+              if Myreplicator::Loader.transfer_completed? metadata            
+                Myreplicator::Loader.incremental_load metadata
+                Myreplicator::Loader.cleanup metadata
               end
 
             end
@@ -231,10 +262,10 @@ module Myreplicator
     # being loaded is completed
     ##
     def self.transfer_completed? metadata
-      Kernel.p "===== transfer_completed? metadata ====="
-      Kernel.p ({:export_id => metadata.export_id,
-                              :file => metadata.export_path,
-      :job_type => "transporter"})
+      #Kernel.p "===== transfer_completed? metadata ====="
+      #Kernel.p ({:export_id => metadata.export_id,
+      #                        :file => metadata.export_path,
+      #:job_type => "transporter"})
       if Log.completed?(:export_id => metadata.export_id,
                         :file => metadata.export_path,
                         :job_type => "transporter")
@@ -283,26 +314,26 @@ module Myreplicator
         files << ExportMetadata.new(:metadata_path => json_file)
       end
       result = []
-      Kernel.p files
+      #Kernel.p files
       files.each do |file|
-        puts "<<<<<<<<<<<<<<<<"
+        #puts "<<<<<<<<<<<<<<<<"
         job = Export.where("id = #{file.export_id}").first
-        puts "<<<<<<<<<<<<<<<<"
-        Kernel.p job
-        puts "<<<<<<<<<<<<<<<<"
-        puts "&&&&&&&&&&&&&&&&&&&&&&&&&&"
-        Kernel.p file
-        puts "&&&&&&&&&&&&&&&&&&&&&&&&&&"
+        #puts "<<<<<<<<<<<<<<<<"
+        #Kernel.p job
+        #puts "<<<<<<<<<<<<<<<<"
+        #puts "&&&&&&&&&&&&&&&&&&&&&&&&&&"
+        #Kernel.p file
+        #puts "&&&&&&&&&&&&&&&&&&&&&&&&&&"
         #if job.state == "transport_completed"
-          result << file
+        result << file
         #end
-        puts "^^^^^^^^^^^^^^^^^^^^^^^^^^"
-        Kernel.p result
-        puts "^^^^^^^^^^^^^^^^^^^^^^^^^^"
+        #puts "^^^^^^^^^^^^^^^^^^^^^^^^^^"
+        #Kernel.p result
+        #puts "^^^^^^^^^^^^^^^^^^^^^^^^^^"
       end
-      puts "<<<<<<<<<<<<<<<<"
+      #puts "<<<<<<<<<<<<<<<<"
       
-      puts "<<<<<<<<<<<<<<<<"
+      #puts "<<<<<<<<<<<<<<<<"
       return result
     end
 
@@ -313,9 +344,9 @@ module Myreplicator
     ##
     def self.clear_older_files metadata
       files = Loader.metadata_files
-      Kernel.p "===== clear old files ====="
-      Kernel.p metadata
-      Kernel.p files
+      #Kernel.p "===== clear old files ====="
+      #Kernel.p metadata
+      #Kernel.p files
       max_date = DateTime.strptime metadata.export_time
       files.each do |m|
         if metadata.export_id == m.export_id

data/lib/myreplicator/version.rb

@@ -1,3 +1,3 @@
 module Myreplicator
-  VERSION = "1.1.20"
+  VERSION = "1.1.21"
 end

data/lib/transporter/transporter.rb

@@ -52,8 +52,45 @@ module Myreplicator
     # Kicks off parallel download
     ##
     def self.download export
-      Kernel.p "===== 1 ====="
-      parallel_download(completed_files(export))
+      #Kernel.p "===== 1 ====="
+      #parallel_download(completed_files(export))
+      files = completed_files(export)
+      files.each do |f|
+        export = f[:export]
+        filename = f[:file]
+        ActiveRecord::Base.verify_active_connections!
+             ActiveRecord::Base.connection.reconnect!
+     
+             Log.run(:job_type => "transporter", :name => "metadata_file", 
+                     :file => filename, :export_id => export.id ) do |log|
+     
+               sftp = export.sftp_to_source
+               json_file = Transporter.remote_path(export, filename) 
+               json_local_path = File.join(tmp_dir,filename)
+               puts "Downloading #{json_file}"
+               sftp.download!(json_file, json_local_path)
+               metadata = Transporter.metadata_obj(json_local_path)
+               dump_file = metadata.export_path
+               puts metadata.state
+               if metadata.state == "export_completed"
+                 Log.run(:job_type => "transporter", :name => "export_file",
+                         :file => dump_file, :export_id => export.id) do |log|
+                   puts "Downloading #{dump_file}"
+                   local_dump_file = File.join(tmp_dir, dump_file.split("/").last)
+                   sftp.download!(dump_file, local_dump_file)
+                   Transporter.remove!(export, json_file, dump_file)
+                   #export.update_attributes!({:state => 'transport_completed'})
+                   # store back up as well
+                   unless metadata.store_in.blank?
+                     Transporter.backup_files(metadata.backup_path, json_local_path, local_dump_file)
+                   end
+                 end
+               else
+                 # TO DO: remove metadata file of failed export
+                 Transporter.remove!(export, json_file, "")
+               end #if
+             end
+      end
     end
 
     ##

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: myreplicator
 version: !ruby/object:Gem::Version
-  version: 1.1.20
+  version: 1.1.21
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-29 00:00:00.000000000 Z
+date: 2013-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -357,7 +357,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1636857574436666365
+      hash: -1843574403921129739
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -366,7 +366,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1636857574436666365
+      hash: -1843574403921129739
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24