mongify 1.0.1 → 1.1.0

data/lib/mongify/status.rb

@@ -2,8 +2,8 @@ module Mongify
   # This class is responsible for generating progress bars with status of mongify
   class Status
     # List of known notifications
-    NOTIFICATIONS = ['copy_data', 'copy_embedded', 'copy_polymorphic', 'update_references', 'remove_pre_mongified']
-    
+    NOTIFICATIONS = ['copy_data', 'copy_embedded', 'copy_polymorphic', 'update_references', 'remove_pre_mongified', 'set_last_updated_at']
+
     class << self
       #List of all the progress bars.
       def bars
@@ -13,14 +13,14 @@ module Mongify
       def add_bar(name, bar)
         self.bars[name] = bar
       end
-      
+
       # Registers the ActiveSupport::Notifications for Mongify
       def register
         ActiveSupport::Notifications.subscribe(/^mongify\./) do |*args|
           event = ActiveSupport::Notifications::Event.new(*args)
-          
+
           action_name = event.name.split('.', 2)[1]
-          
+
           if Status::NOTIFICATIONS.include?(action_name)
             case event.payload[:action]
             when 'add'
@@ -37,12 +37,12 @@ module Mongify
             UI.warn("Unknown Notification Event #{action_name}")
           end
         end
-        
+
         # Unregisters from {ActiveSupport::Notifications}
         def unregister
           ActiveSupport::Notifications.unsubscribe(/^mongify\./)
         end
-        
+
         # Publish an notification event
         # This will publish the event as an mongify.[name]
         # @param [String] name Name of the notification
@@ -55,5 +55,5 @@ module Mongify
         end
       end
     end
-  end 
+  end
 end

data/lib/mongify/translation.rb

@@ -1,5 +1,6 @@
 require 'mongify/translation/printer'
 require 'mongify/translation/process'
+require 'mongify/translation/sync'
 module Mongify
   #
   # Actually runs the translation from sql to no sql
@@ -12,7 +13,7 @@ module Mongify
   #     column "created_at", :datetime
   #     column "updated_at", :datetime
   #   end
-  #   
+  #
   #   table "posts" do
   #     column "id", :key
   #     column "title", :string
@@ -22,7 +23,7 @@ module Mongify
   #     column "created_at", :datetime
   #     column "updated_at", :datetime
   #   end
-  #   
+  #
   #   table "comments", :embed_in => :posts, :on => :post_id do
   #     column "id", :key
   #     column "body", :text
@@ -31,13 +32,13 @@ module Mongify
   #     column "created_at", :datetime
   #     column "updated_at", :datetime
   #   end
-  #   
+  #
   #   table "preferences", :embed_in => :users, :as => :object do
   #     column "id", :key
   #     column "user_id", :integer, :references => "users"
   #     column "notify_by_email", :boolean
   #   end
-  #   
+  #
   #   table "notes", :embed_in => true, :polymorphic => 'notable' do
   #     column "id", :key
   #     column "user_id", :integer, :references => "users"
@@ -47,11 +48,12 @@ module Mongify
   #     column "created_at", :datetime
   #     column "updated_at", :datetime
   #   end
-  # 
-  
+  #
+
   class Translation
     include Printer
     include Process
+    include Sync
     class << self
       # Returns an instance of a translation object
       # Takes a location of a translation file
@@ -60,7 +62,7 @@ module Mongify
         translation.instance_eval(File.read(file_name))
         translation
       end
-      
+
       #Returns an instence of a translation object with a given sql connection layout loaded
       def load(connection)
         raise Mongify::SqlConnectionRequired, "Can only read from Mongify::Database::SqlConnection" unless connection.is_a?(Mongify::Database::SqlConnection)
@@ -76,52 +78,52 @@ module Mongify
         translation
       end
     end
-    
+
     def initialize
       @all_tables = []
     end
-    
+
     # finds table by name
     def find(name)
       all_tables.find{ |t| t.name == name }
     end
-    
+
     # Creates a {Mongify::Database::Table} from the given input and adds it to the list of tables
     def table(table_name, options={}, &block)
       table = Mongify::Database::Table.new(table_name, options, &block)
       add_table(table)
     end
-    
+
     # Adds a {Mongify::Database::Table} to the list of tables
     def add_table(table)
       @all_tables << table
       table
     end
-    
+
     # Returns an array of all tables in the translation
     def all_tables
       @all_tables
     end
-    
+
     # Returns an array of all tables that have not been ingored
     def tables
       all_tables.reject{ |t| t.ignored? || t.polymorphic? }
     end
-    
+
     # Returns an array of all tables that have not been ignored and are just straight copy tables
     def copy_tables
       tables.reject{|t| t.embedded?}
     end
-    
+
     # Returns an array of all tables that have a polymorphic relationship
     def polymorphic_tables
       all_tables.reject{ |t| t.ignored? || !t.polymorphic? }
     end
-    
+
     # Returns an array of all tables that have not been ignored and are to be embedded
     def embed_tables
       tables.reject{|t| !t.embedded?}
     end
-    
+
   end
 end

data/lib/mongify/translation/process.rb

@@ -1,23 +1,15 @@
+require 'mongify/translation/processor_common'
 module Mongify
   class Translation
     #
     # This module does the processing on the translation object
     #
+    include ProcessorCommon
     module Process
-      attr_accessor :sql_connection, :no_sql_connection
       # Does the actual act of processing the translation.
-      # Takes in boht a sql connection and a no sql connection
+      # Takes in both a sql connection and a no sql connection
       def process(sql_connection, no_sql_connection)
-        raise Mongify::SqlConnectionRequired, "Can only read from Mongify::Database::SqlConnection" unless sql_connection.is_a?(Mongify::Database::SqlConnection)
-        raise Mongify::NoSqlConnectionRequired, "Can only write to Mongify::Database::NoSqlConnection" unless no_sql_connection.is_a?(Mongify::Database::NoSqlConnection)
-        
-        self.sql_connection = sql_connection
-        raise SqlConnectionInvalid, "SQL Connection is not valid" unless self.sql_connection.valid?
-        self.no_sql_connection = no_sql_connection
-        raise NoSqlConnectionInvalid, "noSql Connection is not valid" unless self.no_sql_connection.valid?
-        
-        no_sql_connection.ask_to_drop_database if no_sql_connection.forced?
-        
+        prepare_connections(sql_connection, no_sql_connection)
         setup_db_index
         copy_data
         copy_embedded_tables
@@ -26,91 +18,27 @@ module Mongify
         remove_pre_mongified_ids
         nil
       end
-      
+
       #######
       private
       #######
-      
-      # Setups up pre_mongifed_id as an index to speed up lookup performance
-      def setup_db_index
-        self.copy_tables.each do |t|
-          no_sql_connection.create_pre_mongified_id_index(t.name)
-        end
-      end
-      
+
       # Does the straight copy (of tables)
       def copy_data
         self.copy_tables.each do |t|
-          rows = sql_connection.select_rows(t.sql_name)
-          Mongify::Status.publish('copy_data', :size => rows.count, :name => "Copying #{t.name}", :action => 'add')
-          rows.each do |row|
-            no_sql_connection.insert_into(t.name, t.translate(row))
-            Mongify::Status.publish('copy_data')
-          end
-          Mongify::Status.publish('copy_data', :action => 'finish')
-        end
-      end
-      
-      # Does a copy of the embedded tables
-      def copy_embedded_tables
-        self.embed_tables.each do |t|
-          rows = sql_connection.select_rows(t.sql_name)
-          Mongify::Status.publish('copy_embedded', :size => rows.count, :name => "Embedding #{t.name}", :action => 'add')
-          rows.each do |row|
-            target_row = no_sql_connection.find_one(t.embed_in, {:pre_mongified_id => row[t.embed_on]})
-            next unless target_row.present?
-            # puts "target_row = #{target_row.inspect}", "---"
-            row, parent_row = t.translate(row, target_row)
-            parent_row ||= {}
-            parent_row.delete("_id")
-            parent_row.delete(t.name.to_s)
-            #puts "parent_row = #{parent_row.inspect}", "---"
-            row.delete(t.embed_on)
-            row.merge!(fetch_reference_ids(t, row))
-            row.delete('pre_mongified_id')
-            save_function_call = t.embedded_as_object? ? '$set' : '$addToSet'
-            no_sql_connection.update(t.embed_in, target_row['_id'], append_parent_object({save_function_call => {t.name => row}}, parent_row))
-            Mongify::Status.publish('copy_embedded')
-          end
-          Mongify::Status.publish('copy_embedded', :action => 'finish')
-        end
-      end
-      
-      # Moves over polymorphic data
-      def copy_polymorphic_tables
-        self.polymorphic_tables.each do |t|
-          polymorphic_id_col, polymorphic_type_col = "#{t.polymorphic_as}_id", "#{t.polymorphic_as}_type"
-          rows = sql_connection.select_rows(t.sql_name)
-          Mongify::Status.publish('copy_polymorphic', :size => rows.count, :name => "Polymorphicizing #{t.name}", :action => 'add')
-          rows.each do |row|
-            
-            #If no data is in the column, skip importing
-            
-            if (row[polymorphic_type_col])
-              table_name = row[polymorphic_type_col].tableize            
-              new_id = no_sql_connection.get_id_using_pre_mongified_id(table_name, row[polymorphic_id_col])
-            end
-            
-            row = t.translate(row)
-            row[polymorphic_id_col] = new_id if new_id
-            row.merge!(fetch_reference_ids(t, row))
-            row.delete('pre_mongified_id')
-            
-            if t.embedded? && table_name
-              row.delete(polymorphic_id_col)
-              row.delete(polymorphic_type_col)
-              save_function_call = t.embedded_as_object? ? '$set' : '$addToSet'
-              no_sql_connection.update(table_name, new_id, {save_function_call => {t.name => row}})
-            else
-              no_sql_connection.insert_into(t.name, row)
+          sql_connection.select_rows(t.sql_name) do |rows, page, total_pages|
+            Mongify::Status.publish('copy_data', :size => rows.count, :name => "Copying #{t.name} (#{page}/#{total_pages})", :action => 'add')
+            insert_rows = []
+            rows.each do |row|
+              insert_rows << t.translate(row)
+              Mongify::Status.publish('copy_data')
             end
-            
-            Mongify::Status.publish('copy_polymorphic')
+            no_sql_connection.insert_into(t.name, insert_rows) unless insert_rows.empty?
+            Mongify::Status.publish('copy_data', :action => 'finish')
           end
-          Mongify::Status.publish('copy_polymorphic', :action => 'finish')
         end
       end
-      
+
       # Updates the reference ids in the no sql database
       def update_reference_ids
         self.copy_tables.each do |t|
@@ -125,42 +53,7 @@ module Mongify
           Mongify::Status.publish('update_references', :action => 'finish')
         end
       end
-      
-      # Fetches the new _id from a collection
-      def fetch_reference_ids(table, row)
-        attributes = {}
-        table.reference_columns.each do |c|
-          new_id = nil
-          if row[c.name.to_s].is_a?(Array)
-            new_id = []
-            row[c.name.to_s].each do |old_id|
-              new_id << no_sql_connection.get_id_using_pre_mongified_id(c.references.to_s, old_id)
-            end
-          else
-            new_id = no_sql_connection.get_id_using_pre_mongified_id(c.references.to_s, row[c.name.to_s])
-          end
-          attributes.merge!(c.name => new_id) unless new_id.nil?
-        end
-        attributes
-      end
-      
-      # Removes 'pre_mongiifed_id's from all collection
-      def remove_pre_mongified_ids
-        self.copy_tables.each do |t| 
-          Mongify::Status.publish('remove_pre_mongified', :size => 1, :name => "Removing pre_mongified_id #{t.name}", :action => 'add')
-          no_sql_connection.remove_pre_mongified_ids(t.name)
-          Mongify::Status.publish('remove_pre_mongified', :action => 'finish')
-        end
-      end
-      
-      # Used to append parent object values to an embedded update call
-      def append_parent_object(object, parent)
-        return object if parent.blank?
-        object["$set"] = object.has_key?('$set') ? object["$set"].merge(parent) : parent
-        object
-      end
-      
-      
+
     end
   end
 end

data/lib/mongify/translation/processor_common.rb

@@ -0,0 +1,132 @@
+module Mongify
+  class Translation
+    #
+    # This module does the processing on the translation object
+    #
+    module ProcessorCommon
+      attr_accessor :sql_connection, :no_sql_connection
+
+      #########
+      protected
+      #########
+
+      # Prepares connections for process & sync
+      def prepare_connections(sql_connection, no_sql_connection)
+        raise Mongify::SqlConnectionRequired, "Can only read from Mongify::Database::SqlConnection" unless sql_connection.is_a?(Mongify::Database::SqlConnection)
+        raise Mongify::NoSqlConnectionRequired, "Can only write to Mongify::Database::NoSqlConnection" unless no_sql_connection.is_a?(Mongify::Database::NoSqlConnection)
+
+        self.sql_connection = sql_connection
+        raise SqlConnectionInvalid, "SQL Connection is not valid" unless self.sql_connection.valid?
+        self.no_sql_connection = no_sql_connection
+        raise NoSqlConnectionInvalid, "noSql Connection is not valid" unless self.no_sql_connection.valid?
+
+        no_sql_connection.ask_to_drop_database if no_sql_connection.forced?
+
+      end
+
+      # Setups up pre_mongifed_id as an index to speed up lookup performance
+      def setup_db_index
+        self.copy_tables.each do |t|
+          no_sql_connection.create_pre_mongified_id_index(t.name)
+        end
+      end
+
+      # Does a copy of the embedded tables
+      def copy_embedded_tables
+        self.embed_tables.each do |t|
+          rows = sql_connection.select_rows(t.sql_name)
+          Mongify::Status.publish('copy_embedded', :size => rows.count, :name => "Embedding #{t.name}", :action => 'add')
+          rows.each do |row|
+            target_row = no_sql_connection.find_one(t.embed_in, {:pre_mongified_id => row[t.embed_on]})
+            next unless target_row.present?
+            row, parent_row, unset_keys = t.translate(row, target_row)
+            parent_row ||= {}
+            parent_row.delete("_id")
+            parent_row.delete(t.name.to_s)
+            unset_keys ||= {}
+            row.delete(t.embed_on)
+            row.merge!(fetch_reference_ids(t, row))
+            row.delete('pre_mongified_id')
+            save_function_call = t.embedded_as_object? ? '$set' : '$addToSet'
+            no_sql_connection.update(t.embed_in, target_row['_id'], append_parent_object({save_function_call => {t.name => row}}, parent_row, unset_keys))
+            Mongify::Status.publish('copy_embedded')
+          end
+          Mongify::Status.publish('copy_embedded', :action => 'finish')
+        end
+      end
+
+      # Moves over polymorphic data
+      def copy_polymorphic_tables
+        self.polymorphic_tables.each do |t|
+          polymorphic_id_col, polymorphic_type_col = "#{t.polymorphic_as}_id", "#{t.polymorphic_as}_type"
+          rows = sql_connection.select_rows(t.sql_name)
+          Mongify::Status.publish('copy_polymorphic', :size => rows.count, :name => "Polymorphicizing #{t.name}", :action => 'add')
+          rows.each do |row|
+
+            #If no data is in the column, skip importing
+
+            if (row[polymorphic_type_col])
+              table_name = row[polymorphic_type_col].tableize
+              new_id = no_sql_connection.get_id_using_pre_mongified_id(table_name, row[polymorphic_id_col])
+            end
+
+            row = t.translate(row)
+            row[polymorphic_id_col] = new_id if new_id
+            row.merge!(fetch_reference_ids(t, row))
+            row.delete('pre_mongified_id')
+
+            if t.embedded? && table_name
+              row.delete(polymorphic_id_col)
+              row.delete(polymorphic_type_col)
+              save_function_call = t.embedded_as_object? ? '$set' : '$addToSet'
+              no_sql_connection.update(table_name, new_id, {save_function_call => {t.name => row}})
+            else
+              no_sql_connection.insert_into(t.name, row)
+            end
+
+            Mongify::Status.publish('copy_polymorphic')
+          end
+          Mongify::Status.publish('copy_polymorphic', :action => 'finish')
+        end
+      end
+
+      # Fetches the new _id from a collection
+      def fetch_reference_ids(table, row)
+        attributes = {}
+        table.reference_columns.each do |c|
+          new_id = nil
+          if row[c.name.to_s].is_a?(Array)
+            new_id = []
+            row[c.name.to_s].each do |old_id|
+              new_id << no_sql_connection.get_id_using_pre_mongified_id(c.references.to_s, old_id)
+            end
+          else
+            new_id = no_sql_connection.get_id_using_pre_mongified_id(c.references.to_s, row[c.name.to_s])
+          end
+          attributes.merge!(c.name => new_id) unless new_id.nil?
+        end
+        attributes
+      end
+
+      # Removes 'pre_mongiifed_id's from all collection
+      def remove_pre_mongified_ids
+        self.copy_tables.each do |t|
+          Mongify::Status.publish('remove_pre_mongified', :size => 1, :name => "Removing pre_mongified_id #{t.name}", :action => 'add')
+          no_sql_connection.remove_pre_mongified_ids(t.name)
+          Mongify::Status.publish('remove_pre_mongified', :action => 'finish')
+        end
+      end
+
+      # Used to append parent object values to an embedded update call
+      def append_parent_object(object, parent, unset_keys = {})
+        return object if parent.blank?
+        object["$set"] = object.has_key?('$set') ? object["$set"].merge(parent) : parent
+        unless unset_keys.empty?
+          object["$unset"] = object.has_key?('$unset') ? object["$unset"].merge(unset_keys) : unset_keys
+        end
+        object
+      end
+
+    end
+  end
+end