mongify 1.0.1 → 1.1.0

data/spec/mongify/translation/printer_spec.rb

@@ -13,11 +13,11 @@ describe Mongify::Translation do
       @table2.column('id', :key)
       @translation.add_table(@table2)
     end
-    
+
     subject{@translation}
 
     it "should output correctly" do
-      subject.print.should == <<-EOF 
+      subject.print.should == <<-EOF
 table "users" do
 \tcolumn "first_name", :string
 \tcolumn "age", :integer

data/spec/mongify/translation/process_spec.rb

@@ -8,15 +8,15 @@ describe Mongify::Translation::Process do
     @translation = Mongify::Translation.new
     Mongify::Configuration.out_stream = nil
   end
-  
+
   it "validates a sqlconnection" do
     lambda { @translation.process('bad param', 'bad param2') }.should raise_error(Mongify::SqlConnectionRequired)
   end
-  
+
   it "should require a NoSqlConnection" do
     lambda { @translation.process(@sql_connection, 'bad param2') }.should raise_error(Mongify::NoSqlConnectionRequired)
   end
-  
+
   describe "process" do
     before(:each) do
       @translation.stub(:remove_pre_mongified_ids)
@@ -44,7 +44,7 @@ describe Mongify::Translation::Process do
       @translation.should_receive(:remove_pre_mongified_ids)
       @translation.process(@sql_connection, @no_sql_connection)
     end
-    
+
     it "should add pre_mongified_id index to database" do
       tables = [stub(:name => 'users')]
       @translation.stub(:copy_tables).and_return(tables)
@@ -52,13 +52,13 @@ describe Mongify::Translation::Process do
       @translation.process(@sql_connection, @no_sql_connection)
     end
   end
-  
+
   it "should ask_to_drop_database if mongodb_connection is forced" do
     @no_sql_connection.should_receive(:forced?).and_return(true)
     @no_sql_connection.should_receive(:ask_to_drop_database).and_return(false)
     @translation.process(@sql_connection, @no_sql_connection)
   end
-  
+
   context "fetch_reference_ids" do
     it "should get correct information" do
       @no_sql_connection = mock()
@@ -81,37 +81,38 @@ describe Mongify::Translation::Process do
       @translation.send(:fetch_reference_ids, @table, {'user_ids' => [1, 2]}).should == {'user_ids' => [500, 501]}
     end
   end
-  
+
   context "processing actions" do
     before(:each) do
-      @sql_connection = mock(:select_rows => [{'first_name'=> 'Timmy', 'last_name' => 'Zuza', 'preference_id' => 1}])
+      @sql_connection.stub(:select_paged_rows).and_return([{'first_name'=> 'Timmy', 'last_name' => 'Zuza', 'preference_id' => 1}])
       @translation.stub(:sql_connection).and_return(@sql_connection)
-  
+
       @no_sql_connection = mock()
       @translation.stub(:no_sql_connection).and_return(@no_sql_connection)
-      
+
       @table = mock(:translate => {'first_name'=> 'Timmy', 'last_name' => 'Zuza', 'preference_id' => 1},
-                    :name => 'users', 
-                    :embedded? => false, 
+                    :name => 'users',
+                    :embedded? => false,
                     :sql_name => 'users')
-                    
+
       @translation.stub(:tables).and_return([@table])
     end
-    
+
     context "copy_data" do
       it "should call translate on the tables" do
-        @no_sql_connection.should_receive(:insert_into).with("users", {"last_name"=>"Zuza", "preference_id"=>1, "first_name"=>"Timmy"}).and_return(true)
+        @no_sql_connection.should_receive(:insert_into).with("users", [{"last_name"=>"Zuza", "preference_id"=>1, "first_name"=>"Timmy"}]).and_return(true)
         @translation.send(:copy_data)
       end
       it "should allow rename of table" do
         @table.stub(:name).and_return('people')
-        @no_sql_connection.should_receive(:insert_into).with("people", {"last_name"=>"Zuza", "preference_id"=>1, "first_name"=>"Timmy"}).and_return(true)
+        @no_sql_connection.should_receive(:insert_into).with("people", [{"last_name"=>"Zuza", "preference_id"=>1, "first_name"=>"Timmy"}]).and_return(true)
         @translation.send(:copy_data)
       end
     end
-    
+
     context "copy_embed_tables" do
       before(:each) do
+        @sql_connection.stub(:select_rows).and_return([{'first_name'=> 'Timmy', 'last_name' => 'Zuza', 'preference_id' => 1}])
         @target_table = mock(:name => 'posts', :embedded? => false, :sql_name => 'posts')
         @embed_table = mock(:translate => {}, :name => 'comments', :embedded? => true, :embed_on => 'post_id', :embed_in => 'posts', :embedded_as_object? => false, :sql_name => 'comments')
         @no_sql_connection.stub(:find_one).and_return({'_id' => 500})
@@ -148,29 +149,35 @@ describe Mongify::Translation::Process do
         @no_sql_connection.should_receive(:update).with("posts", 500, {"$set"=>{"notes"=>{'first_name' => 'bob'}}})
         @translation.send(:copy_embedded_tables)
       end
-      
+
       context "parent modification" do
+        it 'should unset fields deleted in the parent row' do
+          @embed_table = mock(:translate => [{}, {'email' => 'true'}, {'field_1' => '1'}], :name => 'preferences', :sql_name => 'preferences', :embedded? => true, :embed_on => 'post_id', :embed_in => 'posts', :embedded_as_object? => true)
+          @translation.stub(:tables).and_return([@target_table, @embed_table])
+          @no_sql_connection.should_receive(:update).with("posts", 500, {"$set"=>{"preferences"=>{}, "email"=>"true"}, "$unset"=>{'field_1' => '1'}})
+          @translation.send(:copy_embedded_tables)
+        end
         it "should work with embedded objects" do
-          @embed_table = mock(:translate => [{}, {'email' => 'true'}], :name => 'preferences', :sql_name => 'preferences', :embedded? => true, :embed_on => 'post_id', :embed_in => 'posts', :embedded_as_object? => true)
+          @embed_table = mock(:translate => [{}, {'email' => 'true'}, {}], :name => 'preferences', :sql_name => 'preferences', :embedded? => true, :embed_on => 'post_id', :embed_in => 'posts', :embedded_as_object? => true)
           @translation.stub(:tables).and_return([@target_table, @embed_table])
           @no_sql_connection.should_receive(:update).with("posts", 500, {"$set"=>{"preferences"=>{}, "email"=>"true"}})
           @translation.send(:copy_embedded_tables)
         end
         it "should work with embedded arrays" do
-          @embed_table = mock(:translate => [{}, {'email' => 'true'}], :name => 'preferences', :sql_name => 'preferences', :embedded? => true, :embed_on => 'post_id', :embed_in => 'posts', :embedded_as_object? => false)
+          @embed_table = mock(:translate => [{}, {'email' => 'true'}, {}], :name => 'preferences', :sql_name => 'preferences', :embedded? => true, :embed_on => 'post_id', :embed_in => 'posts', :embedded_as_object? => false)
           @translation.stub(:tables).and_return([@target_table, @embed_table])
           @no_sql_connection.should_receive(:update).with("posts", 500, {"$addToSet"=>{"preferences"=>{}}, "$set" => {"email"=>"true"}})
           @translation.send(:copy_embedded_tables)
         end
         it "should not set embedded attribute in parent" do
-          @embed_table = mock(:translate => [{'first_name' => 'joe'}, {'email' => 'true', 'comments' => [{'first_name' => 'bob'}]}], :name => 'comments', :sql_name => 'comments', :embedded? => true, :embed_on => 'post_id', :embed_in => 'posts', :embedded_as_object? => false)
+          @embed_table = mock(:translate => [{'first_name' => 'joe'}, {'email' => 'true', 'comments' => [{'first_name' => 'bob'}]}, {}], :name => 'comments', :sql_name => 'comments', :embedded? => true, :embed_on => 'post_id', :embed_in => 'posts', :embedded_as_object? => false)
           @translation.stub(:tables).and_return([@target_table, @embed_table])
           @no_sql_connection.should_receive(:update).with("posts", 500, {"$addToSet" => {"comments" => {"first_name" => "joe"}}, "$set" => {"email" => "true"}})
           @translation.send(:copy_embedded_tables)
         end
       end
     end
-    
+
     context "append_parent_object" do
       before(:each) do
         @parent = {"preference" => "email"}
@@ -188,8 +195,17 @@ describe Mongify::Translation::Process do
         @obj = {"$setAppend" => {"existing" => "true"}}
         @translation.send(:append_parent_object, @obj, @parent).should == {"$setAppend" => {"existing" => "true"}}
       end
+      it "should not unset values if no unset_key is given" do
+        @obj = {"$setAppend" => {"existing" => "true"}}
+        @translation.send(:append_parent_object, @obj, @parent, {}).should == {"$set" => {"preference" => "email"}, "$setAppend" => {"existing" => "true"}}
+      end
+      it "should unset values if unset_keys are given" do
+        @obj = {"$set" => {"existing" => "true"}}
+        @translation.send(:append_parent_object, @obj, @parent, {'field_1' => '1'}).should ==
+            {"$set" => {"existing" => "true", "preference" => "email"}, '$unset' => {'field_1' => '1'}}
+      end
     end
-    
+
     context "update_reference_ids" do
       it "should work correctly" do
         @no_sql_connection.should_receive(:select_rows).and_return([{'_id' => 100, 'user_id' => 1}, {'_id'=> 101, 'user_id' => 2}])
@@ -206,32 +222,32 @@ describe Mongify::Translation::Process do
         @translation.send(:update_reference_ids)
       end
     end
-    
+
     context "copy_polymorphic_tables" do
       before(:each) do
-        @ref_table = mock(:name => 'user_accounts', 
+        @ref_table = mock(:name => 'user_accounts',
                           :embedded? => false,
                           :ignored? => false,
                           :sql_name => 'user_accounts')
         @translation.stub(:find).with('user_accounts').and_return([@ref_table])
-        
+
         @sql_connection.stub(:select_rows).with('comments').and_return([{'commentable_id' => 1, 'commentable_type' => 'UserAccount', 'data' => 'good'}])
         @no_sql_connection.stub(:get_id_using_pre_mongified_id).with('user_accounts', 1).and_return(500)
       end
       context "embedded" do
         it "should work correctly" do
            @table = mock(:translate => {'data' => 123},
-                          :name => 'comments', 
+                          :name => 'comments',
                           :embedded? => true,
                           :polymorphic_as => 'commentable',
-                          :polymorphic? => true, 
+                          :polymorphic? => true,
                           :ignored? => false,
                           :embedded_as_object? => false,
                           :sql_name => 'comments',
                           :reference_columns => [])
 
             @translation.stub(:all_tables).and_return([@table])
-          
+
           @no_sql_connection.should_receive(:update).with('user_accounts', 500, {'$addToSet' => {'comments' => {'data' => 123}}})
           @translation.send(:copy_polymorphic_tables)
         end
@@ -239,10 +255,10 @@ describe Mongify::Translation::Process do
       context "not embedded" do
         it "should work" do
           @table = mock(:translate => {'data' => 123, 'commentable_type' => 'UserAccount', 'commentable_id' => 1},
-                          :name => 'comments', 
+                          :name => 'comments',
                           :embedded? => false,
                           :polymorphic_as => 'commentable',
-                          :polymorphic? => true, 
+                          :polymorphic? => true,
                           :ignored? => false,
                           :embedded_as_object? => false,
                           :sql_name => 'comments',
@@ -255,10 +271,10 @@ describe Mongify::Translation::Process do
         end
         it "should copy even if there is no polymorphic data" do
           @table = mock(:translate => {'data' => 123, 'commentable_type' => nil, 'commentable_id' => nil},
-                          :name => 'comments', 
+                          :name => 'comments',
                           :embedded? => false,
                           :polymorphic_as => 'commentable',
-                          :polymorphic? => true, 
+                          :polymorphic? => true,
                           :ignored? => false,
                           :embedded_as_object? => false,
                           :sql_name => 'comments',
@@ -272,7 +288,7 @@ describe Mongify::Translation::Process do
         end
       end
     end
-    
+
     context "remove_pre_mongified_ids" do
       it "should remove_pre_mongified_ids on no_sql_connection" do
         @no_sql_connection.should_receive(:remove_pre_mongified_ids).with(anything)

data/spec/mongify/translation/sync_spec.rb

@@ -0,0 +1,184 @@
+require 'spec_helper'
+
+describe Mongify::Translation::Sync do
+  before(:each) do
+    DatabaseGenerator.clear_mongodb
+    @sql_connection = DatabaseGenerator.sqlite_connection
+    @no_sql_connection = DatabaseGenerator.mongo_connection
+    @translation = Mongify::Translation.new
+    Mongify::Configuration.out_stream = nil
+  end
+
+  it "validates a sqlconnection" do
+    lambda { @translation.sync('bad param', 'bad param2') }.should raise_error(Mongify::SqlConnectionRequired)
+  end
+
+  it "should require a NoSqlConnection" do
+    lambda { @translation.sync(@sql_connection, 'bad param2') }.should raise_error(Mongify::NoSqlConnectionRequired)
+  end
+
+  describe "sync" do
+    before(:each) do
+      @translation.stub(:setup_sync_table)
+      @translation.stub(:setup_db_index)
+      @translation.stub(:sync_data)
+      @translation.stub(:set_last_updated_at)
+      @translation.stub(:sync_update_reference_ids)
+      @translation.stub(:copy_embedded_tables)
+    end
+    it "should create sync helper table if it doesn't exist" do
+      @translation.should_receive(:setup_sync_table)
+      @translation.sync(@sql_connection, @no_sql_connection)
+    end
+    it "should setup index on pre_mongify_id" do
+      @translation.should_receive(:setup_db_index)
+      @translation.sync(@sql_connection, @no_sql_connection)
+    end
+    it "should call copy_data" do
+      @translation.should_receive(:sync_data)
+      @translation.sync(@sql_connection, @no_sql_connection)
+    end
+    it "should call set_last_updated_at to mark synced data in the source" do
+      @translation.should_receive(:set_last_updated_at)
+      @translation.sync(@sql_connection, @no_sql_connection)
+    end
+    it "should call sync_update_reference_ids" do
+      @translation.should_receive(:sync_update_reference_ids)
+      @translation.sync(@sql_connection, @no_sql_connection)
+    end
+    it "should call copy_embedded_tables" do
+      @translation.should_receive(:copy_embedded_tables)
+      @translation.sync(@sql_connection, @no_sql_connection)
+    end
+  end
+
+  context "syncing actions" do
+    before(:each) do
+      @sql_connection = mock(:select_rows => [{'first_name'=> 'Timmy', 'last_name' => 'Zuza', 'preference_id' => 1}])
+      @translation.stub(:sql_connection).and_return(@sql_connection)
+
+      @no_sql_connection = mock()
+      @translation.stub(:no_sql_connection).and_return(@no_sql_connection)
+
+      @table = mock(:translate => {'first_name'=> 'Timmy', 'last_name' => 'Zuza', 'preference_id' => 1},
+                    :name => 'users',
+                    :embedded? => false,
+                    :sql_name => 'users')
+
+      @translation.stub(:tables).and_return([@table])
+    end
+
+    context "SyncHelperMigrator" do
+      it "should create a table with index in the up dir" do
+        migrator = Mongify::Translation::Sync::SyncHelperMigrator.new
+        helper = Mongify::Translation::Sync::SYNC_HELPER_TABLE
+        t = mock({:string => 1, :datetime => 1})
+        t.should_receive(:string).with(:table_name)
+        t.should_receive(:datetime).with(:last_updated_at)
+        migrator.stub(:create_table).and_yield(t)
+        migrator.should_receive(:create_table).with(helper, :id => false)
+        migrator.stub(:add_index)
+        migrator.should_receive(:add_index).with(helper, :table_name)
+        migrator.up
+      end
+    end
+
+    context "setup_sync_table" do
+      before(:each) do
+        @helper = Mongify::Translation::Sync::SYNC_HELPER_TABLE
+        @query = "SELECT count(*) FROM #{@helper}"
+        @sql_connection.stub(:execute).with(@query).and_return(5)
+        @sql_connection.should_receive(:execute).with(@query)
+        @translation.stub(:copy_tables).and_return([mock(:sql_name => 'table1')])
+      end
+
+      it "should create sync helper table if it doesn't exist" do
+        @sql_connection.stub(:execute).with(@query).and_raise
+        migrator = mock(:up)
+        Mongify::Translation::Sync::SyncHelperMigrator.stub(:new).and_return(migrator)
+        migrator.should_receive(:up)
+        @translation.stub(:copy_tables).and_return([])
+        @translation.send(:setup_sync_table)
+      end
+
+      it "should find rows for existing table representatives" do
+        @sql_connection.stub(:count).and_return(1)
+        @sql_connection.should_receive(:count).with(@helper, "table_name = 'table1'")
+        @translation.send(:setup_sync_table)
+      end
+
+      it "should insert new records for non existing table representatives" do
+        @sql_connection.stub(:count).and_return(0)
+        @sql_connection.should_receive(:count).with(@helper, "table_name = 'table1'")
+        insert_query = "INSERT INTO #{@helper} (table_name, last_updated_at) VALUES ('table1', '1970-01-01')"
+        @sql_connection.stub(:execute).with(insert_query)
+        @sql_connection.should_receive(:execute).with(insert_query)
+        @translation.send(:setup_sync_table)
+      end
+    end
+
+    context "set_last_updated_at" do
+      it "should update last_updated_at timestamp for each table that generated sync data" do
+        @translation.stub(:copy_tables).and_return([mock(:sql_name => 'table1')])
+        @translation.max_updated_at = {'table1' => {'max_updated_at_id' => 1, 'key_column' => 'id'}}
+        helper = Mongify::Translation::Sync::SYNC_HELPER_TABLE
+        query = "UPDATE #{helper} SET last_updated_at = (SELECT updated_at FROM table1 WHERE id = '1') WHERE table_name = 'table1'"
+        @sql_connection.stub(:execute).with(query)
+        @sql_connection.should_receive(:execute).with(query)
+        @translation.send(:set_last_updated_at)
+      end
+      it "should not update last_updated_at timestamp for each table that did not generate any sync data" do
+        @translation.stub(:copy_tables).and_return([mock(:sql_name => 'table1')])
+        @translation.max_updated_at = {}
+        @translation.send(:set_last_updated_at)
+        @translation.max_updated_at = {'table1' => {'key_column' => 'id'}}
+        @translation.send(:set_last_updated_at)
+      end
+    end
+
+    context "sync_data" do
+      it "should upsert rows that match the new/updated query, mark them as drafts and compute the max updated at" do
+        helper = Mongify::Translation::Sync::SYNC_HELPER_TABLE
+        t = mock(:sql_name => 'table1', :name => 'table1')
+        @translation.stub(:copy_tables).and_return([t])
+        t1, t2 = Time.new(1980).to_s, Time.new(2000).to_s
+        rows = [{"id" => 1, "updated_at" => t1}, {"id" => 2, "updated_at" => t2}]
+        t.stub(:translate).twice.and_return({'pre_mongified_id' => 1, 'updated_at' => t1}, {'pre_mongified_id' => 2, 'updated_at' => t2})
+
+        query = "SELECT t.* FROM table1 t, #{helper} u WHERE t.updated_at > u.last_updated_at AND u.table_name = 'table1'"
+        @sql_connection.stub(:select_by_query).and_return(rows)
+        @sql_connection.should_receive(:select_by_query).with(query)
+
+        draft = Mongify::Translation::Sync::DRAFT_KEY
+
+        @no_sql_connection.stub(:upsert).with('table1', {'pre_mongified_id' => 1, 'updated_at' => t1, draft => true}).and_return(true)
+        @no_sql_connection.stub(:upsert).with('table1', {'pre_mongified_id' => 2, 'updated_at' => t2, draft => true}).and_return(true)
+        @no_sql_connection.should_receive(:upsert).twice
+
+        t.stub(:key_column).and_return(mock({name: 'id'}))
+
+        @translation.send(:sync_data)
+
+        @translation.max_updated_at.should == {'table1' => {'max_updated_at_id' => 2, 'key_column' => 'id'}}
+
+      end
+    end
+
+    context "sync_update_reference_ids" do
+      it "should delete the draft key" do
+        t = mock(:name => 'table1')
+        @translation.stub(:copy_tables).and_return([t])
+        query = {Mongify::Translation::Sync::DRAFT_KEY => true}
+        row = mock
+        row.stub(:[]).with("_id").and_return(1)
+        @no_sql_connection.stub(:select_by_query).and_return([row])
+        @no_sql_connection.should_receive(:select_by_query).with('table1', query)
+        @translation.stub(:fetch_reference_ids).and_return({})
+        @translation.should_receive(:fetch_reference_ids).with(t, row)
+        @no_sql_connection.should_receive(:update).with('table1', 1, {"$unset" => query})
+        @translation.send(:sync_update_reference_ids)
+      end
+    end
+
+  end
+end

data/spec/mongify/translation_spec.rb

@@ -5,7 +5,7 @@ describe Mongify::Translation do
     @file_path = File.expand_path(File.dirname(__FILE__) + '/../files/translation.rb')
     @translation = Mongify::Translation.parse(@file_path)
   end
-  
+
   context "self.load" do
     it "should require connection" do
       lambda {Mongify::Translation.load}.should raise_error(ArgumentError)
@@ -33,13 +33,13 @@ describe Mongify::Translation do
       end
     end
   end
-  
+
   context "parsed content" do
     context "tables" do
       it "should have 4 tables" do
         @translation.should have(4).tables
       end
-      
+
       it "should setup 'comments'" do
         table = @translation.tables.find{|t| t.name == 'comments'}
         table.should_not be_nil
@@ -47,23 +47,23 @@ describe Mongify::Translation do
       end
     end
   end
-  
+
   context "find" do
     before(:each) do
       @user_table = mock(:name => 'users')
       @translation.stub(:all_tables).and_return([mock(:name => 'comments'),
-                                                 @user_table, 
+                                                 @user_table,
                                                  mock(:name => 'posts')])
     end
     it "should work" do
-      
+
       @translation.find('users').should == @user_table
     end
     it "should return nil if nothing is found" do
       @translation.find('apples').should be_nil
     end
   end
-  
+
   context "tables reference" do
     before(:each) do
       @copy_table = mock(:name => 'users', :ignored? => false, :embedded? => false, :polymorphic? => false)
@@ -93,7 +93,7 @@ describe Mongify::Translation do
       @translation.polymorphic_tables.should == [@polymorphic_table]
     end
   end
-  
+
   context "add_table" do
     before(:each) do
       @table = Mongify::Database::Table.new("users")