mongify 1.0.1 → 1.1.0

data/lib/mongify/database/sql_connection.rb

@@ -3,9 +3,9 @@ module Mongify
     #
     # Sql connection configuration
     #
-    # 
+    #
     # Basic format should look something like this:
-    # 
+    #
     #   sql_connection do
     #     adapter   "mysql"
     #     host      "localhost"
@@ -14,7 +14,7 @@ module Mongify
     #     database  "my_database"
     #   end
     # Possible attributes:
-    # 
+    #
     #   adapter
     #   host
     #   database
@@ -23,13 +23,15 @@ module Mongify
     #   port
     #   encoding
     #   socket
-    # 
+    #   batch_size
+    #
     class SqlConnection < Mongify::Database::BaseConnection
-      
+
       # List of required fields to bulid a valid sql connection
       REQUIRED_FIELDS = %w{host adapter database}
-      
-      def initialize(options=nil)
+
+      def initialize(options={})
+        options['batch_size'] ||= 10000
         super(options)
       end
 
@@ -37,7 +39,7 @@ module Mongify
       def setup_connection_adapter
         ActiveRecord::Base.establish_connection(self.to_hash)
       end
-      
+
       # Returns true or false depending if the record is valid
       def valid?
         return false unless @adapter
@@ -48,20 +50,20 @@ module Mongify
         end
         false
       end
-      
+
       # Returns true or false depending if the connction actually talks to the database server.
       def has_connection?
         setup_connection_adapter
         connection.send(:connect) if ActiveRecord::Base.connection.respond_to?(:connect)
         true
       end
-      
+
       # Returns the active_record connection
       def connection
         return nil unless has_connection?
         ActiveRecord::Base.connection
       end
-      
+
       # Returns all the tables in the database server
       def tables
         return nil unless has_connection?
@@ -72,16 +74,43 @@ module Mongify
       def columns_for(table_name)
         self.connection.columns(table_name)
       end
-      
+
       # Returns an array with hash values of all the records in a given table
-      def select_rows(table_name)
-        self.connection.select_all("SELECT * FROM #{table_name}")
+      def select_rows(table_name, &block)
+        return self.connection.select_all("SELECT * FROM #{table_name}") unless block_given?
+
+        row_count = count(table_name);
+        pages = (row_count.to_f/batch_size).ceil
+        (1..pages).each do |page|
+          rows = select_paged_rows(table_name, batch_size, page)
+          yield rows, page, pages
+        end
+
+      end
+
+      def select_paged_rows(table_name, batch_size, page)
+        connection.select_all("SELECT * FROM #{table_name} LIMIT #{batch_size} OFFSET #{(page - 1) * batch_size}")
+      end
+
+      # Returns an array with hash values of the records in a given table specified by a query
+      def select_by_query(query)
+        self.connection.select_all(query)
+      end
+
+      def count(table_name, where = nil)
+        q = "SELECT COUNT(*) FROM #{table_name}"
+        q = "#{q} WHERE #{where}" if where
+        self.connection.select_value(q).to_i
+      end
+
+      def execute(query)
+        self.connection.execute(query)
       end
 
       #######
       private
       #######
-      # Used to check if this is a sqlite connection 
+      # Used to check if this is a sqlite connection
       def sqlite_adapter?
         @adapter && (@adapter.downcase == 'sqlite' || @adapter.downcase == 'sqlite3')
       end

data/lib/mongify/database/table.rb

@@ -1,125 +1,130 @@
 module Mongify
   module Database
     #
-    #  A representation of a sql table and how it should map to a no_sql collection 
+    #  A representation of a sql table and how it should map to a no_sql collection
     #
     # ==== Structure
-    # 
+    #
     # Structure for defining a table is as follows:
     #   table "table_name", {options} do
     #     # columns go here...
     #   end
-    # 
+    #
     # ==== Options
-    # 
+    #
     # Table Options are as follow:
     #   table "table_name"                                        # Does a straight copy of the table
     #   table "table_name", :embed_in => 'users'                  # Embeds table_name into users, assuming a user_id is present in table_name.
     #                                                             # This will also assume you want the table embedded as an array.
-    # 
+    #
     #   table "table_name",                                       # Embeds table_name into users, linking it via a owner_id
     #         :embed_in => 'users',                               # This will also assume you want the table embedded as an array.
-    #         :on => 'owner_id'                                   
-    # 
+    #         :on => 'owner_id'
+    #
     #   table "table_name",                                       # Embeds table_name into users as a one to one relationship
     #         :embed_in => 'users',                               # This also assumes you have a user_id present in table_name
     #         :on => 'owner_id',                                  # You can also specify both :on and :as options when embedding
-    #         :as => 'object'                                     # NOTE: If you rename the owner_id column, make sure you 
+    #         :as => 'object'                                     # NOTE: If you rename the owner_id column, make sure you
     #                                                             # update the :on to the new column name
-    # 
-    # 
+    #
+    #
     #   table "table_name", :rename_to => 'my_table'              # This will allow you to rename the table as it's getting process
     #                                                             # Just remember that columns that use :reference need to
     #                                                             # reference the new name.
-    # 
+    #
     #   table "table_name", :ignore => true                       # This will ignore the whole table (like it doesn't exist)
     #                                                             # This option is good for tables like: schema_migrations
-    # 
+    #
     #   table "table_name",                                       # This allows you to specify the table as being polymorphic
     #         :polymorphic => 'notable',                          # and provide the name of the polymorphic relationship.
     #         :embed_in => true                                   # Setting embed_in => true allows the relationship to be
     #                                                             # embedded directly into the parent class.
     #                                                             # If you do not embed it, the polymorphic table will be copied in to
     #                                                             # MongoDB and the notable_id will be updated to the new BSON::ObjectID
-    # 
+    #
     #   table "table_name" do                                     # A table can take a before_save block that will be called just
     #     before_save do |row|                                    # before the row is saved to the no sql database.
     #       row.admin = row.delete('permission').to_i > 50        # This gives you the ability to do very powerful things like:
     #     end                                                     # Moving records around, renaming records, changing values in row based on
     #   end                                                       # some values! Checkout Mongify::Database::DataRow to learn more
-    # 
-    # 
+    #
+    #
     #   table "preferences", :embed_in => "users" do               # As of version 0.2, embedded tables with a before_save will take an
     #     before_save do |pref_row, user_row|                      # extra argument which is the parent row of the embedded table.
     #       user_row.email_me = pref_row.delete('email_me')        # This gives you the ability to move things from an embedded table row
     #     end                                                      # to the parent row.
     #   end
-    # 
+    #
 
     class Table
-      
+
       attr_accessor :name, :sql_name
       attr_reader :options, :columns
-      
+
       def initialize(sql_name, options={}, &block)
         @columns = []
         @column_lookup = {}
         @options = options.stringify_keys
         self.sql_name = sql_name
-        
+
         self.instance_exec(&block) if block_given?
-        
+
         import_columns
-        
+
         self
       end
-      
+
       # Returns the no_sql collection name
       def name
         @name ||= @options['rename_to']
         @name ||= self.sql_name
       end
-      
+
       # Returns true if table is ignored
       def ignored?
         @options['ignore']
       end
-      
+
       # Returns true if table is marked as polymorphic
       def polymorphic?
         !!@options['polymorphic']
       end
-      
+
       # Returns the name of the polymorphic association
       def polymorphic_as
         @options['polymorphic'].to_s
       end
-      
+
       # Add a Database Column to the table
       # This expects to get a {Mongify::Database::Column} or it will raise {Mongify::DatabaseColumnExpected} otherwise
       def add_column(column)
         raise Mongify::DatabaseColumnExpected, "Expected a Mongify::Database::Column" unless column.is_a?(Mongify::Database::Column)
         add_and_index_column(column)
       end
-      
+
       # Lets you build a column in the table
       def column(name, type=nil, options={})
         options, type = type, nil if type.is_a?(Hash)
         type = type.to_sym if type
         add_and_index_column(Mongify::Database::Column.new(name, type, options))
       end
-      
+
       # Returns the column if found by the sql_name
       def find_column(name)
         return nil unless (index = @column_lookup[name.to_s.downcase.to_sym])
         @columns[index]
       end
-      
+
       # Returns a array of Columns which reference other columns
       def reference_columns
-        @columns.reject{ |c| !c.referenced? } 
+        @columns.reject{ |c| !c.referenced? }
+      end
+
+      # Returns the column of type :key
+      def key_column
+        @columns.find{ |c| c.type == :key }
       end
-      
+
       # Returns a translated row
       # Takes in a hash of values
       def translate(row, parent=nil)
@@ -130,13 +135,13 @@ module Mongify
         end
         run_before_save(new_row, parent)
       end
-      
-      
+
+
       # Returns the name of the embed_in collection
       def embed_in
         @options['embed_in'].to_s unless @options['embed_in'].nil?
       end
-      
+
       # Returns the type of embed it will be [object or array]
       def embed_as
         return nil unless embedded?
@@ -148,58 +153,69 @@ module Mongify
       def embedded_as_object?
         embed_as == 'object'
       end
-      
+
       # Returns true if this is an embedded table
       def embedded?
         embed_in.present?
       end
-      
+
       # Returns the name of the target column to embed on
       def embed_on
         return nil unless embedded?
         (@options['on'] || "#{@options['embed_in'].to_s.singularize}_id").to_s
       end
-      
+
       # Used to save a block to be ran after the row has been processed but before it's saved to the no sql database
       def before_save(&block)
         @before_save_callback = block
       end
-      
+
       #Used to remove any before save filter
       def remove_before_save_filter!
         @before_save_callback = nil
       end
-            
+
       #######
       private
       #######
-      
+
       # Runs the before save
       # Returns: a new modified row
       def run_before_save(row, parent=nil)
         parentrow = Mongify::Database::DataRow.new(parent) unless parent.nil?
         datarow = Mongify::Database::DataRow.new(row)
+
+        # don't allow deletion of pre_mongified_id, sync needs it!
+        pre_mongified_id = row['pre_mongified_id']
         @before_save_callback.call(datarow, parentrow) unless @before_save_callback.nil?
-        if parentrow            
-          [datarow.to_hash, parentrow.to_hash]
+        new_row = datarow.to_hash
+        new_row['pre_mongified_id'] = pre_mongified_id if pre_mongified_id
+
+        if parentrow
+          parentrow_hash = parentrow.to_hash
+          unsets = parent.keys.inject({}) do |unset_keys, key|
+            unset_keys[key] = '1' unless parentrow_hash.has_key?(key)
+            unset_keys
+          end
+          [new_row, parentrow_hash, unsets]
         else
-          datarow.to_hash
+          new_row
         end
       end
-      
+
       # Indexes the column on the sql_name and adds column to the array
       def add_and_index_column(column)
         @column_lookup[column.sql_name.downcase.to_sym] = @columns.size
         @columns << column
         column
       end
-      
+
       # Imports colunms that are sent in via the options['columns']
       def import_columns
         return unless import_columns = @options.delete('columns')
         import_columns.each { |c| add_column(c) }
       end
-      
+
     end
   end
 end

data/lib/mongify/exceptions.rb

@@ -1,17 +1,17 @@
 module Mongify
   # Base Mongify Error
   class MongifyError < RuntimeError; end
-  
+
   # Not Implemented Error from Mongify
   class NotImplementedMongifyError < MongifyError; end
-  
+
   # File Not Found Exception
   class FileNotFound < MongifyError; end
   # Raise when configuration file is missing
   class ConfigurationFileNotFound < FileNotFound; end
   # Raised when Translation file is missing
   class TranslationFileNotFound < FileNotFound; end
-  
+
   # Basic Configuration Error Exception
   class ConfigurationError < MongifyError; end
   # Raise when a sqlConnection is required but not given
@@ -22,13 +22,13 @@ module Mongify
   class NoSqlConnectionRequired < ConfigurationError; end
   # Raised when a NoSqlConfiguration is invalid?
   class NoSqlConnectionInvalid < ConfigurationError; end
-  
+
   # Raised when a Mongify::Database::Column is expected but not given
   class DatabaseColumnExpected < ConfigurationError; end
 
   # Raised when application has no root folder set
   class RootMissing < MongifyError; end
-  
+
   # Raised when an invalid option is passed via CLI
   class InvalidOption < MongifyError; end
 end

data/lib/mongify/progressbar.rb

@@ -8,7 +8,7 @@
 # You can redistribute it and/or modify it under the terms
 # of Ruby's license.
 #
-# This has been modified by 
+# This has been modified by
 #   Andrew Kalek
 #   Anlek Consulting
 #   http://anlek.com
@@ -48,8 +48,8 @@ module Mongify
     # Formatting for the actual bar
     def fmt_bar
       bar_width = do_percentage * @terminal_width / 100
-      sprintf("|%s%s|", 
-              @bar_mark * bar_width, 
+      sprintf("|%s%s|",
+              @bar_mark * bar_width,
               " " *  (@terminal_width - bar_width))
     end
 
@@ -65,9 +65,9 @@ module Mongify
 
     # Formatting for file transfer
     def fmt_stat_for_file_transfer
-      if @finished_p then 
+      if @finished_p then
         sprintf("%s %s %s", bytes, transfer_rate, elapsed)
-      else 
+      else
         sprintf("%s %s %s", bytes, transfer_rate, eta)
       end
     end
@@ -76,7 +76,7 @@ module Mongify
     def fmt_title
       @title[0,(@title_width - 1)] + ":"
     end
-    
+
     # Formatting for count (x/y)
     def fmt_count
       sprintf('%15s', "(#{@current}/#{@total})")
@@ -106,7 +106,7 @@ module Mongify
     def bytes
       convert_bytes(@current)
     end
-    
+
     # Gets formatting for time
     def format_time (t)
       t = t.to_i
@@ -126,19 +126,19 @@ module Mongify
         sprintf("ETA:  %s", format_time(eta))
       end
     end
-    
+
     # Returns elapsed time
     def elapsed
       elapsed = Time.now - @start_time
       sprintf("Time: %s", format_time(elapsed))
     end
-  
+
     # Returns end of line
     # @return [String] "\n" or "\r"
     def eol
       if @finished_p then "\n" else "\r" end
     end
-    
+
     # Calculates percentage
     # @return [Number] the percentage
     def do_percentage
@@ -148,7 +148,7 @@ module Mongify
         @current  * 100 / @total
       end
     end
-    
+
     # Gets the width of the terminal window
     def get_width
       UI.terminal_helper.output_cols
@@ -157,14 +157,14 @@ module Mongify
     # Draws the bar
     def show
       return unless @out
-      arguments = @format_arguments.map {|method| 
+      arguments = @format_arguments.map {|method|
         method = sprintf("fmt_%s", method)
         send(method)
       }
       line = sprintf(@format, *arguments)
 
       width = get_width
-      if line.length == width - 1 
+      if line.length == width - 1
         @out.print(line + eol)
         @out.flush
       elsif line.length >= width
@@ -188,7 +188,7 @@ module Mongify
       end
 
       # Use "!=" instead of ">" to support negative changes
-      if cur_percentage != prev_percentage || 
+      if cur_percentage != prev_percentage ||
           Time.now - @previous_time >= 1 || @finished_p
         show
       end
@@ -209,7 +209,7 @@ module Mongify
       @finished_p = true
       show
     end
-    
+
     # Returns if the bar is finished
     def finished?
       @finished_p