myreplicator 1.1.31 → 1.1.32

data/db/migrate/20121025191622_create_myreplicator_exports.rb

@@ -15,9 +15,16 @@ class CreateMyreplicatorExports < ActiveRecord::Migration
       t.text :error
       t.boolean :active, :default => true
       t.integer :exporter_pid
+      t.integer :transporter_pid
+      t.integer :loader_pid
       t.datetime :export_started_at, :default => nil
       t.datetime :export_finished_at, :default => nil
+      t.datetime :load_started_at, :default => nil
+      t.datetime :load_finished_at, :default => nil
+      t.datetime :transfer_started_at, :default => nil
+      t.datetime :transfer_finished_at, :default => nil
       t.timestamps
+      t.boolean :analyze_constraints, :default => false
     end  
     add_index :myreplicator_exports, [:source_schema, :destination_schema, :table_name], :unique => true, :name => "unique_index"
   end

data/lib/loader/vertica/vertica_loader.rb

@@ -151,22 +151,21 @@ module Myreplicator
             Loader.cleanup metadata #Remove incremental file
             Kernel.p "===== Remove incremental file ====="
           end
+        elsif get_analyze_constraints(ops) > 0 # check for primary key/unique keys violations
+          exp = Export.find(metadata.export_id)
+          Loader.clear_older_files metadata
+          Loader.cleanup metadata
+          Kernel.p "===== DROP CURRENT TABLE ====="
+          sql = "DROP TABLE IF EXISTS #{options[:db]}.#{options[:destination_schema]}.#{options[:table]} CASCADE;"
+          # run the export. The next time loader runs, it will load the file
+          exp.export
         else
           temp_table = create_temp_table ops
           options[:table] = temp_table
           Kernel.p "===== COPY TO TEMP TABLE #{temp_table} ====="
           vertica_copy options
           exp = Export.find(metadata.export_id)
-          if exp.export_type == 'incremental'
-            options.reverse_merge!(:temp_table => "#{temp_table}")
-            options[:table] = options[:table_name]
-            Kernel.p "===== MERGE ====="
-            vertica_merge options
-            #drop the temp table
-            Kernel.p "===== DROP TEMP TABLE ====="
-            sql = "DROP TABLE IF EXISTS #{options[:db]}.#{options[:destination_schema]}.#{temp_table} CASCADE;"
-            Myreplicator::DB.exec_sql("vertica",sql)
-          elsif exp.export_type == 'all'
+          if exp.export_type == 'all'
             options.reverse_merge!(:temp_table => "#{temp_table}")
             options[:table] = options[:table_name]
             Kernel.p "===== DROP CURRENT TABLE ====="
@@ -175,6 +174,15 @@ module Myreplicator
             sql = "ALTER TABLE #{options[:db]}.#{options[:destination_schema]}.#{options[:temp_table]} RENAME TO \"#{options[:table]}\";"
             Kernel.p sql
             Myreplicator::DB.exec_sql("vertica",sql)
+          elsif exp.export_type == 'incremental'
+            options.reverse_merge!(:temp_table => "#{temp_table}")
+            options[:table] = options[:table_name]
+            Kernel.p "===== MERGE ====="
+            vertica_merge options
+            #drop the temp table
+            Kernel.p "===== DROP TEMP TABLE ====="
+            sql = "DROP TABLE IF EXISTS #{options[:db]}.#{options[:destination_schema]}.#{temp_table} CASCADE;"
+            Myreplicator::DB.exec_sql("vertica",sql)
           end
         end
       end
@@ -270,7 +278,7 @@ module Myreplicator
       def get_mysql_keys mysql_schema_simple_form
         result = []
         mysql_schema_simple_form.each do |col|
-          if col["column_key"] == "PRI"
+          if col["column_key"] == "PRI" || col["column_key"] == "UNI" 
             result << col["column_name"]
           end
         end
@@ -280,7 +288,7 @@ module Myreplicator
       def get_mysql_none_keys mysql_schema_simple_form
         result = []
         mysql_schema_simple_form.each do |col|
-          if col["column_key"].blank?
+          if col["column_key"] != "PRI" && col["column_key"] != "UNI"
             result << col["column_name"]
           end
         end
@@ -413,6 +421,20 @@ module Myreplicator
           end
         end
       end
+      
+      def get_analyze_constraints *args
+        options = args.extract_options!
+        exp = Export.find(options[:export_id])
+        begin
+          if exp.analyze_constraints == 1
+            sql = "SELECT analyze_constraints('#{options[:vertica_db]}.#{options[:vertica_schema]}.#{options[:table]}');"
+            result = Myreplicator::DB.exec_sql("vertica",sql)
+            return result.entries.size
+          end
+        rescue Exception => e
+          puts e.message
+        end        
+      end
 =begin
        def create_all_tables db
          tables = Myreplicator::DB.get_tables(db)

data/lib/loader/vertica/vertica_sql.rb

@@ -7,7 +7,7 @@ module Myreplicator
       sql += "#{options[:table]} ("
 
       index = 1
-      primary_set = false
+      #primary_set = false
 
       options[:columns].each do |column|
         sql += "\"#{column['column_name']}\" "
@@ -15,22 +15,40 @@ module Myreplicator
         sql += data_type(column['data_type'], column['column_type'])
         sql += " "
 
-        if column['column_key'] == "PRI"
-          sql += key(column['column_key']) + " " unless primary_set # set only one primary key
-          primary_set = true
-        end
-
         sql += nullable(column['is_nullable'])
         sql += " "
 
         if index < options[:columns].size
           sql += ", "
-        else
-          sql += ");"
         end
         index += 1
       end
- 
+      
+      # Add primary key
+      primary_set = false
+      options[:columns].each do |column|
+        if column['column_key'] == "PRI"
+          if !primary_set
+            sql += ", PRIMARY KEY (" + "\"#{column['column_name']}\" "
+          else
+            sql += ", " + "\"#{column['column_name']}\" "
+          end
+          primary_set = true
+        end
+      end
+      if primary_set
+        sql += ") "
+      end
+      
+      # Add unique key
+      options[:columns].each do |column|
+        if column['column_key'] == "UNI"
+          sql += ", UNIQUE (" + "\"#{column['column_name']}\" "
+          sql += ") "
+        end
+      end
+      
+      sql += ");"
       puts sql
     
       return sql
@@ -46,13 +64,13 @@ module Myreplicator
     end
  
     def self.data_type type, col_type
-      type = VerticaTypes.convert type, col_type
+      type = Myreplicator::VerticaTypes.convert type, col_type
       result = " #{type} "
       return result
     end
 
     def self.key col_key
-      col_key = VerticaTypes.convert_key col_key
+      col_key = Myreplicator::VerticaTypes.convert_key col_key
       return  "#{col_key} "
     end
   end

data/lib/myreplicator/version.rb

@@ -1,3 +1,3 @@
 module Myreplicator
-  VERSION = "1.1.31"
+  VERSION = "1.1.32"
 end

data/test/dummy/log/development.log

@@ -8868,3 +8868,5 @@ Connecting to database specified by database.yml
 Connecting to database specified by database.yml
 Connecting to database specified by database.yml
 Connecting to database specified by database.yml
+Connecting to database specified by database.yml
+Connecting to database specified by database.yml

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: myreplicator
 version: !ruby/object:Gem::Version
-  version: 1.1.31
+  version: 1.1.32
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-15 00:00:00.000000000 Z
+date: 2013-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -359,7 +359,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3491987006208937872
+      hash: -984056559556813446
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -368,7 +368,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3491987006208937872
+      hash: -984056559556813446
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24