mongify 0.1.2 → 0.1.3

data/lib/mongify/status.rb

@@ -1,25 +1,58 @@
 module Mongify
   # This class is responsible for generating progress bars with status of mongify
   class Status
+    # List of known notifications
+    NOTIFICATIONS = ['copy_data', 'copy_embedded', 'copy_polymorphic', 'update_references', 'remove_pre_mongified']
+    
     class << self
+      #List of all the progress bars.
+      def bars
+        @bars ||= {}
+      end
+      #Add a new bar to the list of progress bars
+      def add_bar(name, bar)
+        self.bars[name] = bar
+      end
+      
       # Registers the ActiveSupport::Notifications for Mongify
       def register
-        @bars = {}
         ActiveSupport::Notifications.subscribe(/^mongify\./) do |*args|
           event = ActiveSupport::Notifications::Event.new(*args)
-          # puts event.name
-          mongify, name, action = event.name.split('.', 3)
-          case action
-          when 'start'
-            @bars[name] = ProgressBar.new(name.humanize, event.payload[:size] || 0)
-          when 'inc'
-            @bars[name].try(:inc)
-          when 'finish'
-            @bars[name].try(:finish)
+          
+          action_name = event.name.split('.', 2)[1]
+          
+          if Status::NOTIFICATIONS.include?(action_name)
+            case event.payload[:action]
+            when 'add'
+              self.add_bar(action_name, ProgressBar.new(event.payload[:name], event.payload[:size]))
+            when 'inc'
+              self.bars[action_name].try(:inc)
+            when 'finish'
+              self.bars[action_name].try(:finish)
+            else
+              UI.warn("Unknown Notification Action #{event.payload[:action]}")
+            end
+            #puts event.payload.inspect
           else
-            UI.warn("Unknown Notification Event #{event.name}")
+            UI.warn("Unknown Notification Event #{action_name}")
           end
         end
+        
+        # Unregisters from {ActiveSupport::Notifications}
+        def unregister
+          ActiveSupport::Notifications.unsubscribe(/^mongify\./)
+        end
+        
+        # Publish an notification event
+        # This will publish the event as an mongify.[name]
+        # @param [String] name Name of the notification
+        # @param [Hash] payload to be sent with the notification
+        # @return [nil]
+        def publish(name, payload={})
+          payload.reverse_merge!(:name => name.humanize, :action => 'inc')
+          ActiveSupport::Notifications.instrument("mongify.#{name}", payload)
+          nil
+        end
       end
     end
   end 

data/lib/mongify/translation/process.rb

@@ -5,7 +5,6 @@ module Mongify
     #
     module Process
       attr_accessor :sql_connection, :no_sql_connection
-      
       # Does the actual act of processing the translation.
       # Takes in boht a sql connection and a no sql connection
       def process(sql_connection, no_sql_connection)
@@ -13,12 +12,13 @@ module Mongify
         raise Mongify::NoSqlConnectionRequired, "Can only write to Mongify::Database::NoSqlConnection" unless no_sql_connection.is_a?(Mongify::Database::NoSqlConnection)
         
         self.sql_connection = sql_connection
-        raise "SQL Connection is not valid" unless self.sql_connection.valid?
+        raise SqlConnectionInvalid, "SQL Connection is not valid" unless self.sql_connection.valid?
         self.no_sql_connection = no_sql_connection
-        raise "noSql Connection is not valid" unless self.no_sql_connection.valid?
+        raise NoSqlConnectionInvalid, "noSql Connection is not valid" unless self.no_sql_connection.valid?
         
         no_sql_connection.ask_to_drop_database if no_sql_connection.forced?
         
+        setup_db_index
         copy_data
         copy_embedded_tables
         update_reference_ids
@@ -31,23 +31,32 @@ module Mongify
       private
       #######
       
+      # Setups up pre_mongifed_id as an index to speed up lookup performance
+      def setup_db_index
+        self.copy_tables.each do |t|
+          no_sql_connection.create_pre_mongified_id_index(t.name)
+        end
+      end
+      
       # Does the straight copy (of tables)
       def copy_data
-        ActiveSupport::Notifications.instrument('mongify.copy_data.start', :size => self.copy_tables.count)
         self.copy_tables.each do |t|
-          sql_connection.select_rows(t.sql_name).each do |row|
+          rows = sql_connection.select_rows(t.sql_name)
+          Mongify::Status.publish('copy_data', :size => rows.count, :name => "Copying #{t.name}", :action => 'add')
+          rows.each do |row|
             no_sql_connection.insert_into(t.name, t.translate(row))
+            Mongify::Status.publish('copy_data')
           end
-          ActiveSupport::Notifications.instrument('mongify.copy_data.inc')
+          Mongify::Status.publish('copy_data', :action => 'finish')
         end
-        ActiveSupport::Notifications.instrument('mongify.copy_data.finish')
       end
       
       # Does a copy of the embedded tables
       def copy_embedded_tables
-        ActiveSupport::Notifications.instrument('mongify.copy_embedded_tables.start', :size => self.embed_tables.count)
         self.embed_tables.each do |t|
-          sql_connection.select_rows(t.sql_name).each do |row|
+          rows = sql_connection.select_rows(t.sql_name)
+          Mongify::Status.publish('copy_embedded', :size => rows.count, :name => "Embedding #{t.name}", :action => 'add')
+          rows.each do |row|
             target_row = no_sql_connection.find_one(t.embed_in, {:pre_mongified_id => row[t.embed_on]})
             next unless target_row.present?
             row = t.translate(row)
@@ -56,54 +65,60 @@ module Mongify
             row.delete('pre_mongified_id')
             save_function_call = t.embedded_as_object? ? '$set' : '$addToSet'
             no_sql_connection.update(t.embed_in, target_row['_id'], {save_function_call => {t.name => row}})
+            Mongify::Status.publish('copy_embedded')
           end
-          ActiveSupport::Notifications.instrument('mongify.copy_embedded_tables.inc')
+          Mongify::Status.publish('copy_embedded', :action => 'finish')
         end
-        ActiveSupport::Notifications.instrument('mongify.copy_embedded_tables.finish')
       end
       
       # Moves over polymorphic data
       def copy_polymorphic_tables
-        ActiveSupport::Notifications.instrument('mongify.copy_polymorphic_tables.start', :size => self.polymorphic_tables.count)
         self.polymorphic_tables.each do |t|
           polymorphic_id_col, polymorphic_type_col = "#{t.polymorphic_as}_id", "#{t.polymorphic_as}_type"
-          sql_connection.select_rows(t.sql_name).each do |row|
-            table_name = row[polymorphic_type_col].tableize            
-            new_id = no_sql_connection.get_id_using_pre_mongified_id(table_name, row[polymorphic_id_col])
-            if new_id
-              row = t.translate(row)
-              row.merge!(fetch_reference_ids(t, row))
-              row[polymorphic_id_col] = new_id
-              row.delete('pre_mongified_id')
-              if t.embedded?
-                row.delete(polymorphic_id_col)
-                row.delete(polymorphic_type_col)
-                save_function_call = t.embedded_as_object? ? '$set' : '$addToSet'
-                no_sql_connection.update(table_name, new_id, {save_function_call => {t.name => row}})
-              else
-                no_sql_connection.insert_into(t.name, row)
-              end
+          rows = sql_connection.select_rows(t.sql_name)
+          Mongify::Status.publish('copy_polymorphic', :size => rows.count, :name => "Polymorphicizing #{t.name}", :action => 'add')
+          rows.each do |row|
+            
+            #If no data is in the column, skip importing
+            
+            if (row[polymorphic_type_col])
+              table_name = row[polymorphic_type_col].tableize            
+              new_id = no_sql_connection.get_id_using_pre_mongified_id(table_name, row[polymorphic_id_col])
+            end
+            
+            row = t.translate(row)
+            row[polymorphic_id_col] = new_id if new_id
+            row.merge!(fetch_reference_ids(t, row))
+            row.delete('pre_mongified_id')
+            
+            if t.embedded? && table_name
+              row.delete(polymorphic_id_col)
+              row.delete(polymorphic_type_col)
+              save_function_call = t.embedded_as_object? ? '$set' : '$addToSet'
+              no_sql_connection.update(table_name, new_id, {save_function_call => {t.name => row}})
             else
-              UI.warn "#{table_name} table not found on #{t.sql_name} polymorphic import"
+              no_sql_connection.insert_into(t.name, row)
             end
+            
+            Mongify::Status.publish('copy_polymorphic')
           end
-          ActiveSupport::Notifications.instrument('mongify.copy_polymorphic_tables.inc')
+          Mongify::Status.publish('copy_polymorphic', :action => 'finish')
         end
-        ActiveSupport::Notifications.instrument('mongify.copy_polymorphic_tables.finish')
       end
       
       # Updates the reference ids in the no sql database
       def update_reference_ids
-        ActiveSupport::Notifications.instrument('mongify.update_reference_ids.start', :size => self.tables.size)
-        self.tables.each do |t|
-          no_sql_connection.select_rows(t.name).each do |row|
+        self.copy_tables.each do |t|
+          rows = no_sql_connection.select_rows(t.name)
+          Mongify::Status.publish('update_references', :size => rows.count, :name => "Updating References #{t.name}", :action => 'add')
+          rows.each do |row|
             id = row["_id"]
             attributes = fetch_reference_ids(t, row)
             no_sql_connection.update(t.name, id, {"$set" => attributes}) unless attributes.blank?
+            Mongify::Status.publish('update_references')
           end
-          ActiveSupport::Notifications.instrument('mongify.update_reference_ids.inc')
+          Mongify::Status.publish('update_references', :action => 'finish')
         end
-        ActiveSupport::Notifications.instrument('mongify.update_reference_ids.finish')
       end
       
       # Fetches the new _id from a collection
@@ -118,13 +133,12 @@ module Mongify
       
       # Removes 'pre_mongiifed_id's from all collection
       def remove_pre_mongified_ids
-        ActiveSupport::Notifications.instrument('mongify.remove_pre_mongified_ids.start', :size => self.tables.size)
-        p = ProgressBar.new("Removed pre_mongified_ids", self.copy_tables.count)
         self.copy_tables.each do |t| 
+          Mongify::Status.publish('remove_pre_mongified', :size => 1, :name => "Removing pre_mongified_id #{t.name}", :action => 'add')
           no_sql_connection.remove_pre_mongified_ids(t.name)
-          ActiveSupport::Notifications.instrument('mongify.remove_pre_mongified_ids.inc')
+          Mongify::Status.publish('remove_pre_mongified', :action => 'finish')
+          # Mongify::Status.publish('remove_pre_mongified')
         end
-        ActiveSupport::Notifications.instrument('mongify.remove_pre_mongified_ids.finish')
       end
       
     end

data/lib/mongify/ui.rb

@@ -49,6 +49,14 @@ module Mongify
       def out_stream
         Configuration.out_stream
       end
+      
+      # Creates an instance of HighLine
+      # which lets us figure out width of console
+      # plus a whole lot more
+      # @reutrn [HighLine] instance 
+      def terminal_helper
+        @terminal_helper ||= HighLine.new
+      end
     end
   end
 end

data/lib/mongify/version.rb

@@ -1,4 +1,4 @@
 module Mongify
   # Mongify's Current Version Number
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/mongify.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.add_dependency('mongo', ">= 1.1.5")
   s.add_dependency('bson_ext', ">= 1.1.5")
   s.add_dependency('net-ssh', ">= 2.0")
-  s.add_dependency('progressbar', ">= 0.9")
+  s.add_dependency('highline', '>= 1.6.1')
 
   s.add_development_dependency('rspec', '>= 2.0')
   s.add_development_dependency('rcov', '>= 0.9.9')

data/spec/mongify/database/base_connection_spec.rb

@@ -26,10 +26,10 @@ describe Mongify::Database::BaseConnection do
   end
 
   it "should raise error when trying to call has_connection?" do
-    lambda { @base_connection.has_connection? }.should raise_error(NotImplementedError)
+    lambda { @base_connection.has_connection? }.should raise_error(Mongify::NotImplementedMongifyError)
   end
   it "should raise error when trying to call setup_connection_adapter" do
-    lambda { @base_connection.setup_connection_adapter}.should raise_error(NotImplementedError)
+    lambda { @base_connection.setup_connection_adapter}.should raise_error(Mongify::NotImplementedMongifyError)
   end
 
   it "should raise error on setting unknown variable setting" do

data/spec/mongify/database/column_spec.rb

@@ -122,6 +122,35 @@ describe Mongify::Database::Column do
     end
   end  
   
+  context "as" do
+    subject {Mongify::Database::Column.new('total', :decimal)}
+    it "should default to string" do
+      subject.as.should == 'string'
+    end
+    it "should allow it to be set to integer" do
+      subject.as = 'integer'
+      subject.should be_as_integer
+    end
+    it "should not allow other values" do
+      subject.as = "zuza"
+      subject.as.should == 'string'
+    end
+  end
+  context "scale" do
+    subject {Mongify::Database::Column.new('total', :decimal, :as => 'integer')}
+    it "should be defaulted to 0" do
+      subject.scale.should be_zero
+    end
+    it "should let you set the scale" do
+      subject.scale = 3
+      subject.scale.should == 3
+    end
+    it "should return 0 on invalid input" do
+      subject.scale = 'zuza'
+      subject.scale.should be_zero
+    end
+  end
+  
   context :to_print do
     before(:each) do
       @column = Mongify::Database::Column.new('first_name', :string)
@@ -160,6 +189,10 @@ describe Mongify::Database::Column do
       @column = Mongify::Database::Column.new('id', :key)
       @column.translate(123123).should == {"pre_mongified_id" => 123123}
     end
+    it "should return an integer for pre_mongified_id" do
+      @column = Mongify::Database::Column.new('id', :key)
+      @column.translate('123123').should == {"pre_mongified_id" => 123123}
+    end
   end
   context :type_cast do
     it "should return value if unknown type" do
@@ -212,14 +245,42 @@ describe Mongify::Database::Column do
       before(:each) do
         @column = Mongify::Database::Column.new('price', :decimal)
       end
-      it "should convert numbers to decimal" do
-        @column.send(:type_cast, 101.43).should == BigDecimal.new("101.43")
+      it "should convert numbers to decimal string" do
+        @column.send(:type_cast, 101.43).should == "101.43"
       end
-      it "should convert integers to decimal" do
-        @column.send(:type_cast, 101).should == BigDecimal.new("101.0")
+      it "should convert integers to decimal string" do
+        @column.send(:type_cast, 101).should == "101.0"
       end
-      it "should convert strings to 0.0" do
-        @column.send(:type_cast, 'zuza').should == BigDecimal.new("0")
+      it "should convert strings to 0.0 (string)" do
+        @column.send(:type_cast, 'zuza').should == "0.0"
+      end
+      it "should return a string value" do
+        @column.send(:type_cast, 101.43).should be_a_kind_of String
+      end
+      
+      context "as integer" do
+        before(:each) do
+          @column = Mongify::Database::Column.new('price', :decimal, :as => 'integer')
+          @value = 101.123455
+        end
+        it "should be as_integer" do
+          @column.should be_as_integer
+        end
+        it "should convert number to integer" do
+          @column.send(:type_cast, @value).should == 101
+        end
+        it "should let you specify scale" do
+          @column.scale = 3
+          @column.send(:type_cast, @value).should == 101123
+        end
+        it "should round correctly to specified scale" do
+          @column.scale = 4
+          @column.send(:type_cast, @value).should == 1011235
+        end
+        it "should return an integer value" do
+          @column.scale = 3
+          @column.send(:type_cast, @value).should be_an_integer
+        end
       end
     end
     context :timestamp do

data/spec/mongify/database/no_sql_connection_spec.rb

@@ -116,9 +116,23 @@ describe Mongify::Database::NoSqlConnection do
       end
     end
     
+    it "should create index for pre_mongified_id" do
+      @collection.should_receive(:create_index).with([["pre_mongified_id", Mongo::ASCENDING]]).and_return(true)
+      @mongodb_connection.create_pre_mongified_id_index('users')
+    end
+    
     context "remove_pre_mongified_ids" do
+      before(:each) do
+        @collection.stub(:index_information).and_return('pre_mongified_id_1' => 'something')
+      end
       it "should call update with unset" do
         @collection.should_receive(:update).with({},{'$unset' => {'pre_mongified_id' => 1}}, {:multi=>true})
+        @collection.stub(:drop_index)
+        @mongodb_connection.remove_pre_mongified_ids('users')
+      end
+      it "should drop the index" do
+        @collection.should_receive(:drop_index).with('pre_mongified_id_1')
+        @collection.stub(:update)
         @mongodb_connection.remove_pre_mongified_ids('users')
       end
     end

data/spec/mongify/status_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe Mongify::Status do
+  it "should warn if it's an unknown notification" do
+    Mongify::UI.should_receive(:warn).at_least(1)
+    Mongify::Status.publish('unknwon')
+  end
+  context "publish" do
+    before(:each) do
+      @bar = stub(:title= => '')
+      Mongify::Status.bars.stub(:[]).and_return(@bar)
+    end
+    it "should create a new progress bar" do
+      Mongify::ProgressBar.should_receive(:new)
+      Mongify::Status.publish('copy_data', :size => 100, :action => 'add')
+    end
+    context "on inc" do
+      before(:each) do
+        @bar.stub(:inc)
+      end
+      it "should inc a already created progress bar" do
+        @bar.should_receive(:inc)
+        Mongify::Status.publish('copy_data', :action => 'inc')
+      end
+      it "should inc by default action" do
+        @bar.should_receive(:inc)
+        Mongify::Status.publish('copy_data')
+      end
+    end
+    it "should finish the progress bar" do
+      @bar.should_receive(:finish)
+      Mongify::Status.publish('copy_data', :action => 'finish')
+    end
+  end
+end

data/spec/mongify/translation/process_spec.rb

@@ -23,6 +23,10 @@ describe Mongify::Translation::Process do
       @translation.stub(:copy_data)
       @translation.stub(:copy_embedded_tables)
     end
+    it "should setup index on pre_mongify_id" do
+      @translation.should_receive(:setup_db_index)
+      @translation.process(@sql_connection, @no_sql_connection)
+    end
     it "should call copy_data" do
       @translation.should_receive(:copy_data)
       @translation.process(@sql_connection, @no_sql_connection)
@@ -39,6 +43,13 @@ describe Mongify::Translation::Process do
       @translation.should_receive(:remove_pre_mongified_ids)
       @translation.process(@sql_connection, @no_sql_connection)
     end
+    
+    it "should add pre_mongified_id index to database" do
+      tables = [stub(:name => 'users')]
+      @translation.stub(:copy_tables).and_return(tables)
+      @no_sql_connection.should_receive(:create_pre_mongified_id_index).with('users')
+      @translation.process(@sql_connection, @no_sql_connection)
+    end
   end
   
   it "should ask_to_drop_database if mongodb_connection is forced" do
@@ -47,7 +58,6 @@ describe Mongify::Translation::Process do
     @translation.process(@sql_connection, @no_sql_connection)
   end
   
-  
   context "fetch_reference_ids" do
     it "should get correct information" do
       @no_sql_connection = mock()
@@ -153,8 +163,8 @@ describe Mongify::Translation::Process do
                           :sql_name => 'user_accounts')
         @translation.stub(:find).with('user_accounts').and_return([@ref_table])
         
-        @sql_connection.should_receive(:select_rows).with('comments').and_return([{'commentable_id' => 1, 'commentable_type' => 'UserAccount', 'data' => 'good'}])
-        @no_sql_connection.should_receive(:get_id_using_pre_mongified_id).with('user_accounts', 1).and_return(500)
+        @sql_connection.stub(:select_rows).with('comments').and_return([{'commentable_id' => 1, 'commentable_type' => 'UserAccount', 'data' => 'good'}])
+        @no_sql_connection.stub(:get_id_using_pre_mongified_id).with('user_accounts', 1).and_return(500)
       end
       context "embedded" do
         it "should work correctly" do
@@ -187,10 +197,27 @@ describe Mongify::Translation::Process do
                           :reference_columns => [])
 
           @translation.stub(:all_tables).and_return([@table])
-            
+          @no_sql_connection.should_receive(:get_id_using_pre_mongified_id).with('user_accounts', 1).and_return(500)
           @no_sql_connection.should_receive(:insert_into).with('comments', {'data' => 123, 'commentable_type' => 'UserAccount', 'commentable_id' => 500})
           @translation.send(:copy_polymorphic_tables)
         end
+        it "should copy even if there is no polymorphic data" do
+          @table = mock(:translate => {'data' => 123, 'commentable_type' => nil, 'commentable_id' => nil},
+                          :name => 'comments', 
+                          :embedded? => false,
+                          :polymorphic_as => 'commentable',
+                          :polymorphic? => true, 
+                          :ignored? => false,
+                          :embedded_as_object? => false,
+                          :sql_name => 'comments',
+                          :reference_columns => [])
+
+          @translation.stub(:all_tables).and_return([@table])
+          @sql_connection.should_receive(:select_rows).with('comments').and_return([{'commentable_id' => nil, 'commentable_type' => nil, 'data' => 'good'}])
+          @no_sql_connection.should_receive(:insert_into).with('comments', {'data' => 123, 'commentable_type' => nil, 'commentable_id' => nil})
+          @no_sql_connection.should_receive(:get_id_using_pre_mongified_id).never
+          @translation.send(:copy_polymorphic_tables)
+        end
       end
     end