myreplicator 1.1.11 → 1.1.12

data/app/models/myreplicator/log.rb

@@ -117,10 +117,12 @@ module Myreplicator
     def self.completed? *args
       options = args.extract_options!
       log = Log.where(:export_id => options[:export_id],
-                      :file => options[:export_id],
+                      :file => options[:file],
                       :job_type => options[:job_type]).last
+      Kernel.p "===== transport_complete? log ====="
+      Kernel.p log
       if log.nil?
-        return true
+        return false
       else
         return true if log.state != "running"
       end

data/lib/loader/loader.rb

@@ -32,8 +32,12 @@ module Myreplicator
       initials = []
       incrementals = []
       all_files = Myreplicator::Loader.metadata_files
-
+      
+      Kernel.p "===== all_files ====="
+      Kernel.p all_files 
+      
       all_files.each do |m|
+        Kernel.p m
         if m.export_type == "initial"
           initials << m # Add initial to the list
           all_files.delete(m) # Delete obj from mixed list
@@ -156,9 +160,9 @@ module Myreplicator
     ##
     def self.initial_load metadata
       exp = Export.find(metadata.export_id)
-      Kernel.p "===== unzip ====="
-      Loader.unzip(metadata.filename)
-      metadata.zipped = false
+      #Kernel.p "===== unzip ====="
+      #Loader.unzip(metadata.filename)
+      #metadata.zipped = false
       
       cmd = ImportSql.initial_load(:db => exp.destination_schema,
                                    :filepath => metadata.destination_filepath(tmp_dir))      
@@ -178,8 +182,8 @@ module Myreplicator
     ##
     def self.incremental_load metadata
       exp = Export.find(metadata.export_id)
-      Loader.unzip(metadata.filename)
-      metadata.zipped = false
+      #Loader.unzip(metadata.filename)
+      #metadata.zipped = false
       
       options = {:table_name => exp.table_name, 
         :db => exp.destination_schema,
@@ -227,8 +231,12 @@ module Myreplicator
     # being loaded is completed
     ##
     def self.transfer_completed? metadata
+      Kernel.p "===== transfer_completed? metadata ====="
+      Kernel.p ({:export_id => metadata.export_id,
+                              :file => metadata.export_path,
+      :job_type => "transporter"})
       if Log.completed?(:export_id => metadata.export_id,
-                        :file => metadata.destination_filepath(tmp_dir),
+                        :file => metadata.export_path,
                         :job_type => "transporter")
         return true
       end
@@ -274,7 +282,28 @@ module Myreplicator
       Dir.glob(File.join(tmp_dir, "*.json")).each do |json_file|
         files << ExportMetadata.new(:metadata_path => json_file)
       end
-      return files
+      result = []
+      Kernel.p files
+      files.each do |file|
+        puts "<<<<<<<<<<<<<<<<"
+        job = Export.where("id = #{file.export_id}").first
+        puts "<<<<<<<<<<<<<<<<"
+        Kernel.p job
+        puts "<<<<<<<<<<<<<<<<"
+        puts "&&&&&&&&&&&&&&&&&&&&&&&&&&"
+        Kernel.p file
+        puts "&&&&&&&&&&&&&&&&&&&&&&&&&&"
+        if job.state == "transport_completed"
+          result << file
+        end
+        puts "^^^^^^^^^^^^^^^^^^^^^^^^^^"
+        Kernel.p result
+        puts "^^^^^^^^^^^^^^^^^^^^^^^^^^"
+      end
+      puts "<<<<<<<<<<<<<<<<"
+      a = gets
+      puts "<<<<<<<<<<<<<<<<"
+      return result
     end
 
     ##

data/lib/loader/vertica/vertica_loader.rb

@@ -137,10 +137,12 @@ module Myreplicator
           vertica_copy options
         elsif schema_check[:changed]
           if metadata.export_type == 'initial'
+            Kernel.p "===== schema_check[:changed] ====="
             Loader.clear_older_files metadata  # clear old incremental files
             apply_schema_change(ops, temp_table)
           else
             Loader.cleanup metadata #Remove incremental file
+            Kernel.p "===== Remove incremental file ====="
           end
         else
           temp_table = create_temp_table ops
@@ -354,24 +356,24 @@ module Myreplicator
           # place holder
         end
       end
-
-      # def create_all_tables db
-      #   tables = Loader::SourceDb.get_tables(db)
-      #   sqls = {}
-      #   tables.each do |table|
-      #     puts "Creating #{db}.#{table}"
-      #     sql = "DROP TABLE IF EXISTS #{db}.#{table} CASCADE;"
-      #     VerticaDb::Base.connection.execute sql
-      #     sql = Loader::VerticaLoader.create_table(:vertica_db => "bidw",
-      #     :vertica_table => table,
-      #     :vertica_schema => db,
-      #     :table => table,
-      #     :db => db)
-      #     sqls["#{table}"] = sql
-      #     VerticaDb::Base.connection.execute sql
-      #   end
-      # end
-
+=begin
+       def create_all_tables db
+         tables = Myreplicator::DB.get_tables(db)
+         sqls = {}
+         tables.each do |table|
+           puts "Creating #{db}.#{table}"
+           sql = "DROP TABLE IF EXISTS #{db}.#{table} CASCADE;"
+           VerticaDb::Base.connection.execute sql
+           sql = Loader::VerticaLoader.create_table(:vertica_db => "bidw",
+           :vertica_table => table,
+           :vertica_schema => db,
+           :table => table,
+           :db => db)
+           sqls["#{table}"] = sql
+           VerticaDb::Base.connection.execute sql
+         end
+       end
+=end
     end
   end
 end

data/lib/myreplicator/version.rb

@@ -1,3 +1,3 @@
 module Myreplicator
-  VERSION = "1.1.11"
+  VERSION = "1.1.12"
 end

data/lib/transporter/transporter.rb

@@ -39,8 +39,9 @@ module Myreplicator
     # downloads export files concurrently from multiple sources
     ##
     def self.transfer
-      unique_jobs = Export.where("active = 1").group("source_schema")
-
+      unique_jobs = Export.where("active = 1 and state = 'export_completed'").group("source_schema")
+      Kernel.p "===== unique_jobs ====="
+      Kernel.p unique_jobs 
       unique_jobs.each do |export|
         download export
       end
@@ -51,19 +52,20 @@ module Myreplicator
     # Kicks off parallel download
     ##
     def self.download export
-      parallel_download(export, completed_files(export))
+      Kernel.p "===== 1 ====="
+      parallel_download(completed_files(export))
     end
 
     ##
     # Gathers all files that need to be downloaded
     # Gives the queue to parallelizer library to download in parallel
     ##
-    def self.parallel_download export, files    
+    def self.parallel_download files    
       p = Parallelizer.new(:klass => "Myreplicator::Transporter")
-
-      files.each do |filename|
-        puts filename
-        p.queue << {:params =>[export, filename], :block => download_file}
+    
+      files.each do |f|
+        puts f[:file]
+        p.queue << {:params =>[f[:export], f[:file]], :block => download_file}
       end
 
       p.run 
@@ -103,6 +105,7 @@ module Myreplicator
               local_dump_file = File.join(tmp_dir, dump_file.split("/").last)
               sftp.download!(dump_file, local_dump_file)
               Transporter.remove!(export, json_file, dump_file)
+              export.update_attributes!({:state => 'transport_completed'})
               # store back up as well
               unless metadata.store_in.blank?
                 Transporter.backup_files(metadata.backup_path, json_local_path, local_dump_file)
@@ -144,12 +147,30 @@ module Myreplicator
     def self.completed_files export
       ssh = export.ssh_to_source
       done_files = ssh.exec!(get_done_files(export))
-
-      unless done_files.blank?
-        return done_files.split("\n")
+      if done_files.blank?
+        return []
       end
-
-      return []
+      files = done_files.split("\n")
+      jobs = Export.where("active = 1 and state = 'export_completed' and source_schema = '#{export.source_schema}'")
+      #jobs.each do |j|
+      #  j.update_attributes!({:state => "transporting"})
+      #end
+      result = []
+      files.each do |file|
+        flag = nil
+        jobs.each do |job|
+          if file.include?(job.table_name)
+            flag = job 
+            job.update_attributes!({:state => 'transporting'})
+          end
+        end
+        if flag
+          result << {:file => file, :export => flag}
+        end
+      end
+      Kernel.p "===== done_files ====="
+      Kernel.p result
+      return result
     end
 
     def self.metadata_obj json_path
@@ -177,6 +198,8 @@ module Myreplicator
     # Grep -s used to supress error messages
     ## 
     def self.get_done_files export
+      Kernel.p "===== export ====="
+      Kernel.p export
       cmd = "cd #{Myreplicator.configs[export.source_schema]["ssh_tmp_dir"]}; grep -ls export_completed *.json"
     end