mongify 1.0.1 → 1.1.0

data/lib/mongify/translation/sync.rb

@@ -0,0 +1,112 @@
+require 'mongify/translation/processor_common'
+module Mongify
+  class Translation
+    #
+    # This module does the sync processing on the translation object
+    #
+    include ProcessorCommon
+    module Sync
+      attr_accessor :max_updated_at
+      DRAFT_KEY = "__mongify_sync_draft__"
+      SYNC_HELPER_TABLE = "__mongify_sync_helper__"
+
+      class SyncHelperMigrator < ActiveRecord::Migration
+        def up
+          create_table SYNC_HELPER_TABLE, :id => false do |t|
+            t.string :table_name
+            t.datetime :last_updated_at
+          end
+          add_index SYNC_HELPER_TABLE, :table_name
+        end
+      end
+
+      # Does the actual act of sync processing the translation.
+      # Takes in both a sql connection and a no sql connection
+      def sync(sql_connection, no_sql_connection)
+        prepare_connections(sql_connection, no_sql_connection)
+        setup_sync_table
+        setup_db_index
+        sync_data
+        set_last_updated_at
+        copy_embedded_tables
+        sync_update_reference_ids
+        copy_polymorphic_tables
+        nil
+      end
+
+      #######
+      private
+      #######
+
+      def setup_sync_table
+        # make sure table exists
+        begin
+          self.sql_connection.execute("SELECT count(*) FROM #{SYNC_HELPER_TABLE}")
+        rescue
+          SyncHelperMigrator.new.up
+        end
+        # insert missing records for sync tables
+        self.copy_tables.each do |t|
+          if self.sql_connection.count(SYNC_HELPER_TABLE, "table_name = '#{t.sql_name}'") == 0
+            self.sql_connection.execute("INSERT INTO #{SYNC_HELPER_TABLE} (table_name, last_updated_at) VALUES ('#{t.sql_name}', '1970-01-01')")
+          end
+        end
+      end
+
+      # Does the straight copy (of tables)
+      def sync_data
+        self.copy_tables.each do |t|
+          q = "SELECT t.* FROM #{t.sql_name} t, #{SYNC_HELPER_TABLE} u " +
+            "WHERE t.updated_at > u.last_updated_at AND u.table_name = '#{t.sql_name}'"
+          rows = sql_connection.select_by_query(q)
+          Mongify::Status.publish('copy_data', :size => rows.count, :name => "Syncing #{t.name}", :action => 'add')
+          max_updated_at, max_updated_at_id = Time.new(1970), nil
+          rows.each do |row|
+            row_hash = t.translate(row)
+            updated_at = Time.parse(row['updated_at'])
+            if updated_at > max_updated_at
+              max_updated_at = updated_at
+              max_updated_at_id = row_hash['pre_mongified_id']
+            end
+            no_sql_connection.upsert(t.name, row_hash.merge({DRAFT_KEY => true}))
+            Mongify::Status.publish('copy_data')
+          end
+          (self.max_updated_at ||= {})[t.sql_name] = {'max_updated_at_id' => max_updated_at_id, 'key_column' => t.key_column.name}
+          Mongify::Status.publish('copy_data', :action => 'finish')
+        end
+      end
+
+      # Updates the reference ids in the no sql database
+      def sync_update_reference_ids
+        self.copy_tables.each do |t|
+          rows = no_sql_connection.select_by_query(t.name, {DRAFT_KEY => true})
+          Mongify::Status.publish('update_references', :size => rows.count, :name => "Updating References #{t.name}", :action => 'add')
+          rows.each do |row|
+            id = row["_id"]
+            attributes = fetch_reference_ids(t, row)
+            setter = {"$unset" => {DRAFT_KEY => true}}
+            setter["$set"] = attributes unless attributes.blank?
+            no_sql_connection.update(t.name, id, setter)
+            Mongify::Status.publish('update_references')
+          end
+          Mongify::Status.publish('update_references', :action => 'finish')
+        end
+      end
+
+      # Sets the last updated_at flag so that next sync doesn't unnecessarily copy old data
+      def set_last_updated_at
+        tables = self.copy_tables
+        Mongify::Status.publish('set_last_updated_at', :size => tables.length, :name => "Setting last_updated_at", :action => 'add')
+        tables.each do |t|
+          info = self.max_updated_at[t.sql_name]
+          if info && info['max_updated_at_id']
+            sql_connection.execute("UPDATE #{SYNC_HELPER_TABLE} SET last_updated_at = (SELECT updated_at FROM #{t.sql_name} WHERE #{info['key_column']} = '#{info['max_updated_at_id']}') WHERE table_name = '#{t.sql_name}'")
+          end
+          Mongify::Status.publish('set_last_updated_at')
+        end
+        Mongify::Status.publish('set_last_updated_at', :action => 'finish')
+      end
+
+    end
+  end
+end

data/lib/mongify/ui.rb

@@ -8,39 +8,39 @@ module Mongify
       def puts(msg)
         out_stream.puts(msg) if out_stream
       end
-      
+
       # Outputs to stream using print method
       def print(msg)
         out_stream.print(msg) if out_stream
       end
-    
+
       # Gets input from user
       def gets
         in_stream ? in_stream.gets : ''
       end
-    
+
       # Outputs a question and gets input
       def request(msg)
         print(msg)
         gets.chomp
       end
-    
+
       # Asks a yes or no question and waits for reply
       def ask(msg)
         request("#{msg} [yn] ") == 'y'
       end
-      
+
       # Outputs a Warning (using puts command)
       def warn(msg)
         puts "WARNING: #{msg}"
       end
-      
+
       # Outputs a message and aborts execution of app
       def abort(message)
         UI.puts "PROGRAM HALT: #{message}"
         Kernel.abort message
       end
-      
+
       # Incoming stream
       def in_stream
         Configuration.in_stream
@@ -49,11 +49,11 @@ module Mongify
       def out_stream
         Configuration.out_stream
       end
-      
+
       # Creates an instance of HighLine
       # which lets us figure out width of console
       # plus a whole lot more
-      # @return [HighLine] instance 
+      # @return [HighLine] instance
       def terminal_helper
         @terminal_helper ||= HighLine.new
       end

data/lib/mongify/version.rb

@@ -1,4 +1,4 @@
 module Mongify
   # Mongify's Current Version Number
-  VERSION = "1.0.1"
+  VERSION = "1.1.0"
 end

data/mongify.gemspec

@@ -23,10 +23,12 @@ Gem::Specification.new do |s|
   s.add_development_dependency('cucumber', '>= 0.10')
   s.add_development_dependency('mocha', '>= 0.9.8')
   s.add_development_dependency('yard', '>= 0.5.3')
-  s.add_development_dependency('sqlite3-ruby', '>= 1.3')
-  s.add_development_dependency('mysql2', '~> 0.2.7')
+  s.add_development_dependency('sqlite3', '>= 1.3')
+  s.add_development_dependency('pg', '>= 0.17')
+  s.add_development_dependency('mysql2', '~> 0.3.1')
   s.add_development_dependency('watchr', '>= 0.6')
   s.add_development_dependency('rake')
+  s.add_development_dependency('pry-plus')
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/spec/files/deleting_fields_from_embedding_parent_translation.rb

@@ -0,0 +1,19 @@
+table 'teams' do
+  column 'id', :key
+  column 'name', :string
+  column 'phone', :string
+  column 'created_at', :datetime
+  column 'updated_at', :datetime
+end
+
+table 'coaches', :embed_in => 'teams', :as => 'object', :rename_to => 'coach' do
+  column 'id', :key
+  column 'team_id', :integer, :references => "teams"
+  column 'first_name', :string
+  column 'last_name', :string
+  column 'created_at', :datetime
+  column 'updated_at', :datetime
+  before_save do |row, parent_row|
+    row.write_attribute('phone',parent_row.delete('phone'))
+  end
+end

data/spec/files/embedded_parent_translation.rb

@@ -12,7 +12,7 @@ table "preferences", :embed_in => :users, :as => :array do
 	column "notify_by_email", :boolean
 	column "created_at", :datetime, :ignore => true
 	column "updated_at", :datetime, :ignore => true
-	
+
 	before_save do |row, parent|
 	  parent.notify_by_email = row.delete('notify_by_email')
 	end

data/spec/mongify/cli/application_spec.rb

@@ -7,7 +7,7 @@ describe Mongify::CLI::Application do
       Mongify::Configuration.out_stream = nil
       @application.execute!.should == 0
     end
-    
+
     it "should return a 1 on error" do
       @application = Mongify::CLI::Application.new(["door"])
       Mongify::Configuration.out_stream = nil

data/spec/mongify/cli/options_spec.rb

@@ -52,7 +52,7 @@ describe Mongify::CLI::Options do
     it "should call Configuration.parse" do
       Mongify::Configuration.should_receive(:parse).and_return(Mongify::Configuration.new)
       @options = Mongify::CLI::Options.new(['check', config_file])
-      @options.parse 
+      @options.parse
     end
   end
 end

data/spec/mongify/cli/worker_command_spec.rb

@@ -11,19 +11,19 @@ describe Mongify::CLI::Command::Worker do
     @no_sql_connection.stub(:valid?).and_return(true)
     @no_sql_connection.stub(:has_connection?).and_return(true)
     @config.stub(:no_sql_connection).and_return(@no_sql_connection)
-    
+
     @translation_file = 'spec/files/translation.rb'
-    
+
     Mongify::Translation.stub(:load).and_return(stub(:print => 'worked'))
     @view = mock('view').as_null_object
   end
-  
+
   context "list_commands" do
     it "should return same number as available" do
       Mongify::CLI::Command::Worker.list_commands.size.should == Mongify::CLI::Command::Worker::AVAILABLE_COMMANDS.size
     end
   end
-  
+
   context "check command" do
     before(:each) do
       @command = Mongify::CLI::Command::Worker.new('check', @config)
@@ -42,17 +42,17 @@ describe Mongify::CLI::Command::Worker do
       lambda { @command = Mongify::CLI::Command::Worker.new('check').execute(@view) }.should raise_error(Mongify::ConfigurationFileNotFound)
     end
   end
-  
+
   context "non-existing command" do
      before(:each) do
       @command = Mongify::CLI::Command::Worker.new('unknown')
     end
-    
+
     it "should report error" do
       @view.should_receive(:report_error)
       @command.execute(@view)
     end
-    
+
     it "should output an error" do
       @view.should_receive(:output).with("Unknown action unknown\n\n")
       @command.execute(@view)
@@ -64,22 +64,22 @@ describe Mongify::CLI::Command::Worker do
       @command = Mongify::CLI::Command::Worker.new('translation', @config)
       Mongify::Translation.stub(:load).with(@sql_connection).and_return(stub(:print => 'worked'))
     end
-    
+
     it "should require configuration file" do
       lambda { Mongify::CLI::Command::Worker.new('translation').execute(@view) }.should raise_error(Mongify::ConfigurationFileNotFound)
     end
-    
+
     it "should check sql connection" do
       @command.should_receive(:check_sql_connection).and_return(true)
       @command.execute(@view)
     end
-    
+
     it "should call load on Translation" do
       Mongify::Translation.should_receive(:load).with(@sql_connection).and_return(stub(:print => 'worked'))
       @command.execute(@view)
     end
   end
-  
+
   context "process command" do
     before(:each) do
       @command = Mongify::CLI::Command::Worker.new('process', @config, 'spec/files/translation.rb')
@@ -90,25 +90,54 @@ describe Mongify::CLI::Command::Worker do
       @view.should_receive(:report_success)
       @command.execute(@view)
     end
-    
+
     it "should require config file" do
       lambda { @command = Mongify::CLI::Command::Worker.new('process').execute(@view) }.should raise_error(Mongify::ConfigurationFileNotFound)
     end
-    
+
     it "should require transaction file" do
       lambda { @command = Mongify::CLI::Command::Worker.new('process', @config).execute(@view) }.should raise_error(Mongify::TranslationFileNotFound)
     end
-    
+
     it "should check_connection" do
       @command.should_receive(:check_connections).and_return(true)
       @command.execute(@view)
     end
-    
+
     it "should call process on translation" do
       Mongify::Translation.should_receive(:parse).and_return(mock(:process => true))
       @command.execute(@view)
     end
-    
-    
   end
+
+  context "sync command" do
+    before(:each) do
+      @command = Mongify::CLI::Command::Worker.new('sync', @config, 'spec/files/translation.rb')
+      Mongify::Translation.stub(:parse).and_return(mock(:sync => true))
+    end
+    it "should report success" do
+      @view.should_receive(:report_error).never
+      @view.should_receive(:report_success)
+      @command.execute(@view)
+    end
+
+    it "should require config file" do
+      lambda { @command = Mongify::CLI::Command::Worker.new('sync').execute(@view) }.should raise_error(Mongify::ConfigurationFileNotFound)
+    end
+
+    it "should require transaction file" do
+      lambda { @command = Mongify::CLI::Command::Worker.new('sync', @config).execute(@view) }.should raise_error(Mongify::TranslationFileNotFound)
+    end
+
+    it "should check_connection" do
+      @command.should_receive(:check_connections).and_return(true)
+      @command.execute(@view)
+    end
+
+    it "should call sync on translation" do
+      Mongify::Translation.should_receive(:parse).and_return(mock(:sync => true))
+      @command.execute(@view)
+    end
+  end
+
 end

data/spec/mongify/database/column_spec.rb

@@ -4,41 +4,41 @@ describe Mongify::Database::Column do
   before(:each) do
     @column = Mongify::Database::Column.new('first_name')
   end
-  
+
   it "should have name" do
     @column.name.should == 'first_name'
   end
   it "should have a sql_name" do
     @column.sql_name.should == 'first_name'
   end
-  
+
   it "should allow you to omit the type while giving options" do
     @column = Mongify::Database::Column.new('account_id', :references => 'accounts')
     @column.options.should == {'references' => 'accounts'}
   end
-  
+
   it "should get setup options" do
     @column = Mongify::Database::Column.new('account_id', :integer, :references => 'accounts')
     @column.options.should == {'references' => 'accounts'}
   end
-  
+
   it "should force type to string if nil" do
     @column = Mongify::Database::Column.new('first_name', nil)
     @column.type.should == :string
   end
-  
+
   context "auto_detect!" do
     it "should not auto detect automatically" do
       Mongify::Database::Column.should_receive(:auto_detect).never
       @column = Mongify::Database::Column.new('id', :integer)
       @column.should_not be_key
     end
-    
+
     it "should auto_detect when option is passed in" do
       @column = Mongify::Database::Column.new('id', :integer, :auto_detect => true)
       @column.should be_key
     end
-    
+
     context "id column" do
       before(:each) do
         @col = mock(:sql_name => 'id')
@@ -82,19 +82,19 @@ describe Mongify::Database::Column do
       end
     end
   end
-  
+
   context "key?" do
     it "should be true" do
       @column = Mongify::Database::Column.new('id', :key)
       @column.should be_a_key
     end
-    
+
     it "should be true" do
       @column = Mongify::Database::Column.new('first_name', :string)
       @column.should_not be_a_key
     end
   end
-  
+
   context "rename_to" do
     before(:each) do
       @column = Mongify::Database::Column.new('surname', :string, :rename_to => 'last_name')
@@ -108,12 +108,12 @@ describe Mongify::Database::Column do
     it "should translate to new name" do
       @column.translate('value').should == {'last_name' => 'value'}
     end
-    
+
     it "should rename this" do
       @column = Mongify::Database::Column.new("geoCode_longitude", :string, :rename_to => 'longitude')
       @column.translate(42.2222).should == {'longitude' => '42.2222'}
     end
-    
+
     it "should be renamed" do
       @column.should be_renamed
     end
@@ -122,7 +122,7 @@ describe Mongify::Database::Column do
       col.should_not be_renamed
     end
   end
-  
+
   context "options" do
     it "should allow to be set by name" do
       @column = Mongify::Database::Column.new('first_name')
@@ -134,8 +134,8 @@ describe Mongify::Database::Column do
       @column = Mongify::Database::Column.new('first_name')
       lambda { @column.unknown = "users" }.should raise_error(NoMethodError)
     end
-  end  
-  
+  end
+
   context "as" do
     subject {Mongify::Database::Column.new('total', :decimal)}
     it "should default to string" do
@@ -173,7 +173,7 @@ describe Mongify::Database::Column do
       subject.scale.should be_zero
     end
   end
-  
+
   context :to_print do
     before(:each) do
       @column = Mongify::Database::Column.new('first_name', :string)
@@ -195,14 +195,14 @@ describe Mongify::Database::Column do
       @column.to_print.should == %q{column "first_name", :key, :as => :integer}
     end
   end
-  
+
   context :referenced? do
     it "should be true" do
       @column = Mongify::Database::Column.new('user_id', :integer, :references => 'users')
       @column.should be_a_referenced
     end
   end
-  
+
   context :translate do
     it "should return a hash with the new translation" do
       @column = Mongify::Database::Column.new('first_name', :string)
@@ -213,7 +213,7 @@ describe Mongify::Database::Column do
       @column.should be_ignored
       @column.translate('bob').should == {}
     end
-    
+
     it "should return pre_mongified_id when type is a key" do
       @column = Mongify::Database::Column.new('id', :key)
       @column.translate(123123).should == {"pre_mongified_id" => 123123}
@@ -295,7 +295,7 @@ describe Mongify::Database::Column do
       it "should return a string value" do
         @column.send(:type_cast, 101.43).should be_a_kind_of String
       end
-      
+
       context :integer do
         before(:each) do
           @column = Mongify::Database::Column.new('price', :decimal, :as => :integer)
@@ -387,7 +387,7 @@ describe Mongify::Database::Column do
         @column.send(:type_cast, nil).should == result
         @column.send(:type_cast, "").should == result
       end
-      
+
     end
   end
 end