myreplicator 1.1.57 → 1.1.58

data/app/controllers/myreplicator/home_controller.rb

@@ -19,7 +19,7 @@ module Myreplicator
       @tab = 'home'
       @option = 'errors'
       @exports = Export.where("error is not null").order('source_schema ASC')    
-      @logs = Log.where(:state => 'error').order("started_at DESC") 
+      @logs = Log.where(:state => 'error').order("started_at DESC").limit(200)
     end
 
     def kill

data/app/models/myreplicator/sweeper.rb

@@ -13,12 +13,13 @@ module Myreplicator
       ActiveRecord::Base.configurations.keys.each do |db|
         Myreplicator::VerticaLoader.clean_up_temp_tables(db)
       end
-      folders = ["/home/share/datareplicator",
-        "/home/share/okl/bi_apps/datareplicator/mysqldumps"
-      ]
+      
+      #removing files that are left in the storage for more than 12 hours
+      folders = [
+        "/home/share/datareplicator",
+        "/home/share/okl/bi_apps/datareplicator/mysqldumps"]
         
       folders.each do |folder|
-        #cmd = "find #{folder} -mtime +2"
         cmd = "find #{folder} -mmin +720"
         l = `#{cmd}`
         list = l.split(/\n/)

data/lib/exporter/export_metadata.rb

@@ -1,4 +1,3 @@
-require 'json'
 
 module Myreplicator
   class ExportMetadata

data/lib/exporter/mysql_exporter.rb

@@ -110,7 +110,7 @@ module Myreplicator
     def incremental_export_into_outfile metadata
       unless @export_obj.is_running?
         
-        if @export_obj.export_type == "incremental"
+        if @export_obj.export_type == "incremental" || (@export_obj.export_type == "all" && @export_obj.export_to == "vertica")
           max_value = @export_obj.max_value
           metadata.export_type = "incremental"        
           @export_obj.update_max_val if @export_obj.max_incremental_value.blank?   

data/lib/exporter/sql_commands.rb

@@ -150,7 +150,7 @@ module Myreplicator
           sql = ""
           replaces.each do |k,v|
             if sql.blank?
-              sql = "REPLACE(#{column}, '#{k}', '#{v}')"
+              sql = "REPLACE(\\`#{column}\\`, '#{k}', '#{v}')"
             else
               sql = "REPLACE(#{sql}, '#{k}', '#{v}')"
             end
@@ -159,9 +159,10 @@ module Myreplicator
           end
           result << sql
         else
-          result << column
+          result << "\\`#{column}\\`"
         end
       end
+      Kernel.p result
       return result
     end
 
@@ -199,7 +200,7 @@ module Myreplicator
           sql += "WHERE #{options[:incremental_col]} >= #{options[:incremental_val].to_i - 10000}" #buffer 10000 
         end
       end
-
+      Kernel.p sql
       return sql
     end
 

data/lib/loader/loader.rb

@@ -4,6 +4,7 @@ module Myreplicator
   class Loader
     
     @queue = :myreplicator_load # Provided for Resque
+     
     
     def initialize *args
       options = args.extract_options!
@@ -52,8 +53,9 @@ module Myreplicator
       all_files = Myreplicator::Loader.metadata_files
       
       #Kernel.p "===== all_files ====="
-      #Kernel.p all_files 
-      
+      #Kernel.p all_files
+       
+=begin      
       all_files.each do |m|
         #Kernel.p m
         if m.export_type == "initial"
@@ -70,7 +72,66 @@ module Myreplicator
       end
       
       incrementals = all_files # Remaining are all incrementals
+=end   
+      # adding incremetal loading file to redis
+      files_to_metadata = {}
+      @redis = Redis.new(:host => Settings[:redis][:host], :port => Settings[:redis][:port])
+      @load_set = "myreplicator_load_set"
+      @load_hash = "myreplicator_load_hash"
+      
+      all_files.each do |m|
+        if !(@redis.hexists(@load_hash, m.filepath))
+          @redis.hset(@load_hash, m.filepath, 0)
+          @redis.sadd(@load_set, m.filepath)
+        else
+          if @redis.hget(@load_hash, m.filepath) == 1
+            @redis.hdel(@load_hash, m.filepath)
+          end
+        end
+        files_to_metadata[m.filepath] = m
+      end
+      
+      while @redis.smembers(@load_set).size >= 0
+        filepath = @redis.spop(@load_set)
+              
+        metadata = files_to_metadata[filepath]
+        if metadata.blank?
+          next
+        end
+        if metadata.export_type == "initial"
+          Myreplicator::Log.run(:job_type => "loader",
+          :name => "#{metadata.export_type}_import",
+          :file => metadata.filename,
+          :export_id => metadata.export_id) do |log|
+            if Myreplicator::Loader.transfer_completed? metadata
+              if metadata.export_to == "vertica"
+                Myreplicator::Loader.incremental_load metadata
+              else
+                Myreplicator::Loader.initial_load metadata
+              end
+              Myreplicator::Loader.cleanup metadata
+            end
+          end
+          @redis.hset(@load_hash, metadata.filepath, 1)
+        else #if metadata.export_type == "incremental"
+          #1
+          Myreplicator::Log.run(:job_type => "loader",
+          :name => "incremental_import",
+          :file => metadata.filename,
+          :export_id => metadata.export_id) do |log|
+            if Myreplicator::Loader.transfer_completed? metadata
+              Myreplicator::Loader.incremental_load metadata
+              Myreplicator::Loader.cleanup metadata
+            end
+          end
+          #2
+          @redis.hset(@load_hash, metadata.filepath, 1)
+        end
+        
+      end
+      
       
+=begin      
       #initial_procs = Loader.initial_loads initials
       #parallel_load initial_procs
       initials.each do |metadata|
@@ -106,7 +167,7 @@ module Myreplicator
       end
        #   end # group
       #end # groups
-         
+=end         
     end
     
     def self.parallel_load procs

data/lib/myreplicator/version.rb

@@ -1,3 +1,3 @@
 module Myreplicator
-  VERSION = "1.1.57"
+  VERSION = "1.1.58"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: myreplicator
 version: !ruby/object:Gem::Version
-  version: 1.1.57
+  version: 1.1.58
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-24 00:00:00.000000000 Z
+date: 2013-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -355,7 +355,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -661295928114993181
+      hash: 3425503714195991273
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -364,7 +364,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -661295928114993181
+      hash: 3425503714195991273
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24