mongify 0.1.2 → 0.1.3

data/CHANGELOG.rdoc

@@ -1,3 +1,12 @@
+== 0.1.3 / 21 Feb 2011
+* Made all exception stem from MongifyError and system always raises a child of MongifyError.
+* Brought in the progress bar into source
+* Changed behaviour on storing BigDecimal (now converts into String)
+* Added ability to convert BigDecimal to integer via an :at => 'integer, :scale => 2 settings.
+* Improved Progress Bar display, now it gives you a better feel for what's going on.
+* Added an index on pre_mongified_id to speedup lookup times (Making it 42 times faster on import of embedded tables)
+* Fixed bug in importing of polymorphic tables where associations are nil 
+* Fixed bug where pre_mongified_id would be a string and not an integer (causing no updates of ref_ids)
 == 0.1.2 / 14 Feb 2011
 * More Improvements to README and RDOC
 * Added ability to :force => true on a mongodb_connection, forcing it to drop database before processing.

data/Gemfile.lock

@@ -1,31 +1,31 @@
 PATH
   remote: .
   specs:
-    mongify (0.1.2)
+    mongify (0.1.3)
       activerecord (>= 3.0.3)
       activesupport (>= 3.0.3)
       bson_ext (>= 1.1.5)
+      highline (>= 1.6.1)
       mongo (>= 1.1.5)
       mysql2
       net-ssh (>= 2.0)
-      progressbar (>= 0.9)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    activemodel (3.0.3)
-      activesupport (= 3.0.3)
+    activemodel (3.0.4)
+      activesupport (= 3.0.4)
       builder (~> 2.1.2)
       i18n (~> 0.4)
-    activerecord (3.0.3)
-      activemodel (= 3.0.3)
-      activesupport (= 3.0.3)
+    activerecord (3.0.4)
+      activemodel (= 3.0.4)
+      activesupport (= 3.0.4)
       arel (~> 2.0.2)
       tzinfo (~> 0.3.23)
-    activesupport (3.0.3)
-    arel (2.0.7)
-    bson (1.2.0)
-    bson_ext (1.2.0)
+    activesupport (3.0.4)
+    arel (2.0.8)
+    bson (1.2.2)
+    bson_ext (1.2.2)
     builder (2.1.2)
     cucumber (0.10.0)
       builder (>= 2.1.2)
@@ -36,26 +36,24 @@ GEM
     diff-lcs (1.1.2)
     gherkin (2.3.3)
       json (~> 1.4.6)
+    highline (1.6.1)
     i18n (0.5.0)
     json (1.4.6)
-    mocha (0.9.10)
-      rake
-    mongo (1.2.0)
-      bson (>= 1.2.0)
+    mocha (0.9.12)
+    mongo (1.2.2)
+      bson (>= 1.2.2)
     mysql (2.8.1)
     mysql2 (0.2.6)
     net-ssh (2.1.0)
-    progressbar (0.9.0)
-    rake (0.8.7)
     rcov (0.9.9)
-    rspec (2.4.0)
-      rspec-core (~> 2.4.0)
-      rspec-expectations (~> 2.4.0)
-      rspec-mocks (~> 2.4.0)
-    rspec-core (2.4.0)
-    rspec-expectations (2.4.0)
+    rspec (2.5.0)
+      rspec-core (~> 2.5.0)
+      rspec-expectations (~> 2.5.0)
+      rspec-mocks (~> 2.5.0)
+    rspec-core (2.5.1)
+    rspec-expectations (2.5.0)
       diff-lcs (~> 1.1.2)
-    rspec-mocks (2.4.0)
+    rspec-mocks (2.5.0)
     sqlite3 (1.3.3)
     sqlite3-ruby (1.3.3)
       sqlite3 (>= 1.3.3)

data/README.rdoc

@@ -177,6 +177,8 @@ Table Options are as follow:
     end                                                     # Moving records around, renaming records, changing values in row based on
   end                                                       # some values! Checkout Mongify::Database::DataRow to learn more
 
+More documentation can be found at {Mongify::Database::Table}
+
 === Columns
 
 ==== Structure
@@ -194,7 +196,7 @@ Before we cover the options, you need to know what types of columns are supporte
   :key                  # Columns that are primary keys need to be marked as :key type
   :integer              # Will be converted to a integer
   :float                # Will be converted to a float
-  :decimal              # Will be converted to a BigDecimal
+  :decimal              # Will be converted to a string (due to MongoDB Ruby Drivers not supporting BigDecimal, read details in Mongify::Database::Column under Decimal Storage)
   :string               # Will be converted to a string
   :text                 # Will be converted to a string
   :datetime             # Will be converted to a Time format (DateTime is not currently supported in the Mongo ruby driver)
@@ -215,6 +217,18 @@ Before we cover the options, you need to know what types of columns are supporte
   column "surname", 
          :string, 
          :rename_to => 'last_name'                    # Rename_to allows you to rename the column                                                      
+  
+  <em>For decimal columns you can specify a few options:</em>
+  column "total",                                     # This is a default conversion setting.
+         :decimal,
+         :as => 'string'                              
+         
+  column "total",                                     # You can specify to convert your decimal to integer
+          :decimal,                                   # specifying scale will define how many decimal places to keep
+          :as => 'integer',                           # Example: :scale => 2 will convert 123.4567 to 12346 before saving
+          :scale => 2                                 
+
+More documentation can be found at {Mongify::Database::Column}
 
 == Notes
 
@@ -224,8 +238,12 @@ More testing will be done once the basic functionality is done
 
 
 == TODO
+* Allow deeper embedding
+* Test in Ruby 1.9
+* Test in different databases
 
-* Update user on what is being processed
+== Known Issues
+* Can't do anything to an embedded table
 
 == Development
 

data/lib/mongify.rb

@@ -3,8 +3,9 @@
 #
 require 'active_support/core_ext'
 require 'active_record'
-require 'progressbar'
+require 'highline'
 
+require 'mongify/progressbar'
 require 'mongify/ui'
 require 'mongify/status'
 require 'mongify/exceptions'

data/lib/mongify/cli/application.rb

@@ -26,9 +26,12 @@ module Mongify
         begin
           cmd = @options.parse
           cmd.execute(self)
-        rescue Exception => error
+        rescue MongifyError => error
           $stderr.puts "Error: #{error}"
           report_error
+        rescue Exception => error
+          report_error
+          raise error
         end
         return @status
       end

data/lib/mongify/cli/options.rb

@@ -87,6 +87,8 @@ EOB
       # option parser, ensuring parse_options is only called once     
       def parse_options
         @parsed = true && @parser.parse!(@argv) unless @parsed
+      rescue OptionParser::InvalidOption => er
+        raise Mongify::InvalidOption, er.message, er.backtrace
       end
     end
   end

data/lib/mongify/database/base_connection.rb

@@ -41,12 +41,12 @@ module Mongify
 
       # Used to setup connection, Raises NotImplementedError because it needs to be setup in BaseConnection's children
       def setup_connection_adapter
-        raise NotImplementedError
+        raise NotImplementedMongifyError
       end
 
       # Used to test connection, Raises NotImplementedError because it needs to be setup in BaseConnection's children
       def has_connection?
-        raise NotImplementedError
+        raise NotImplementedMongifyError
       end
 
 

data/lib/mongify/database/column.rb

@@ -19,7 +19,7 @@ module Mongify
     #   :key                  # Columns that are primary keys need to be marked as :key type
     #   :integer              # Will be converted to a integer
     #   :float                # Will be converted to a float
-    #   :decimal              # Will be converted to a BigDecimal
+    #   :decimal              # Will be converted to a string *(you can change default behaviour read below)
     #   :string               # Will be converted to a string
     #   :text                 # Will be converted to a string
     #   :datetime             # Will be converted to a Time format (DateTime is not currently supported in the Mongo ruby driver)
@@ -41,12 +41,46 @@ module Mongify
     #
     #   column "post_id", :integer, :auto_detect => true    # Will run auto detect and make this column a :references => 'posts', :on => 'post_id' for you
     #                                                       # More used when reading a sql database, NOT recommended for use during processing of translation
+    # 
+    # <em>For decimal columns you can specify a few options:</em>
+    #   column "total",                                     # This is a default conversion setting.
+    #          :decimal,
+    #          :as => 'string'                              
     #   
+    #   column "total",                                     # You can specify to convert your decimal to integer
+    #           :decimal,                                   # specifying scale will define how many decimal places to keep
+    #           :as => 'integer',                           # Example: :scale => 2 will convert 123.4567 to 12346 in to mongodb
+    #           :scale => 2
+    # ==== Decimal Storage
+    # 
+    # Unfortunately MongoDB Ruby Drivers doesn't support BigDecimal, so to ensure all data is stored correctly (without losing information)
+    # I've chosen to store as String, however you can overwrite this functionality in one of two ways:
+    # <em>The reason you would want to do this, is to make this searchable via a query.</em>
+    # 
+    # <b>1) You can specify :as => 'integer', :scale => 2</b>
+    #   column "total", :decimal, :as => 'integer', :scale => 2
+    # 
+    #   #It would take a value of 123.456 and store it as an integer of value 12346
+    # 
+    # <b>2) You can specify your own custom conversion by doing a {Mongify::Database::Table#before_save}
+    # 
+    # Example:
+    #   table "invoice" do
+    #     column "name", :string
+    #     column "total", :decimal
+    #     before_save do |row|
+    #       row.total = (BigDecimal.new(row.total) * 1000).round
+    #     end
+    #   end
+    # 
+    # This would take 123.456789 in the total column and convert it to an interger of value 123457 (and in your app you can convert it back to a decimal)
+    # 
+    # *REMEMBER* there is a limit on how big of an integer you can store in BSON/MongoDB (http://bsonspec.org/#/specification)
     class Column
       attr_reader :sql_name, :type, :options
       
       #List of available options for a column
-      AVAILABLE_OPTIONS = ['references', 'ignore', 'rename_to']
+      AVAILABLE_OPTIONS = ['references', 'ignore', 'rename_to', 'as', 'scale']
       
       # Auto detects if a column is an :key column or is a reference column
       def self.auto_detect(column)
@@ -88,7 +122,7 @@ module Mongify
         return {} if ignored?
         case type
         when :key
-          {"pre_mongified_id" => value}
+          {"pre_mongified_id" => value.to_i}
         else
           {"#{name}" => type_cast(value)}
         end
@@ -115,14 +149,20 @@ module Mongify
       def method_missing(meth, *args, &blk)
         method_name = meth.to_s.gsub("=", '')
         if AVAILABLE_OPTIONS.include?(method_name)
-          class_eval <<-EOF
-                          def #{method_name}=(value)
-                            options['#{method_name}'] = value
-                          end
-                          def #{method_name}
-                            options['#{method_name}']
-                          end
-                        EOF
+          unless self.class.method_defined?(method_name.to_sym)
+            class_eval <<-EOF
+                            def #{method_name}=(value)
+                              options['#{method_name}'] = value
+                            end
+                          EOF
+          end
+          unless self.class.method_defined?("#{method_name}=".to_sym)
+            class_eval <<-EOF
+                            def #{method_name}
+                              options['#{method_name}']
+                            end
+                          EOF
+          end
           send(meth, *args, &blk)
         else
           super(meth, *args, &blk)
@@ -154,6 +194,34 @@ module Mongify
         !!self.ignore
       end
       
+      # Used when trying to figure out how to convert a decimal value
+      # @return [String] passed option['as'] or defaults to 'string'
+      def as
+        options['as'] ||= 'string'
+      end
+      # Sets option['as'] to either 'string' or 'integer', defults to 'string' for unknown values
+      # @param [String|Symbol] value, can be either 'string' or 'integer
+      def as=(value)
+        value = value.to_s.downcase
+        options['as'] = ['string', 'integer'].include?(value) ? value : 'string'
+      end
+      # Returns true if :as was passed as integer
+      def as_integer?
+        self.as == 'integer'
+      end
+      
+      # Get the scale option for decimal to integer conversion
+      #   column 'total', :decimal, :as => 'integer', :scale => 3
+      # @return [integer] passed option['scale'] or 0
+      def scale
+        options['scale'] ||= 0
+      end
+      # Set the scale option for decimal to integer conversion
+      #   column 'total', :decimal, :as => 'integer', :scale => 3
+      # @param [Integer] number of decimal places to round off to
+      def scale=(value)
+        options['scale'] = value.to_i
+      end
       #######
       private
       #######
@@ -164,9 +232,15 @@ module Mongify
         case type
           when :string    then value
           when :text      then value
-          when :integer   then value.to_i rescue value ? 1 : 0
+          when :integer   then value.to_i
           when :float     then value.to_f
-          when :decimal   then ActiveRecord::ConnectionAdapters::Column.value_to_decimal(value)
+          when :decimal
+            value = ActiveRecord::ConnectionAdapters::Column.value_to_decimal(value)
+            if as_integer?
+              (value * (10 ** self.scale)).round.to_i
+            else     
+              value.to_s
+            end
           when :datetime  then ActiveRecord::ConnectionAdapters::Column.string_to_time(value)
           when :timestamp then ActiveRecord::ConnectionAdapters::Column.string_to_time(value)
           when :time      then ActiveRecord::ConnectionAdapters::Column.string_to_dummy_time(value)
@@ -176,6 +250,7 @@ module Mongify
           else value
         end
       end
+
       
 
       # runs auto detect (see {Mongify::Database::Column.auto_detect})

data/lib/mongify/database/no_sql_connection.rb

@@ -27,6 +27,7 @@ module Mongify
     class NoSqlConnection < Mongify::Database::BaseConnection
       include Mongo
       
+      
       #Required fields for a no sql connection
       REQUIRED_FIELDS = %w{host database}  
       
@@ -114,10 +115,26 @@ module Mongify
       
       # Removes pre_mongified_id from all records in a given collection
       def remove_pre_mongified_ids(collection_name)
+        drop_mongified_index(collection_name)
         db[collection_name].update({}, { '$unset' => { 'pre_mongified_id' => 1} }, :multi => true)
       end
       
+      # Removes pre_mongified_id from collection
+      # @param [String] collection_name name of collection to remove the index from
+      # @return True if successful
+      # @raise MongoDBError if index isn't found
+      def drop_mongified_index(collection_name)
+        db[collection_name].drop_index('pre_mongified_id_1') if db[collection_name].index_information.keys.include?("pre_mongified_id_1")
+      end
+      
+      # Creates a pre_mongified_id index to ensure
+      # speedy lookup for collections via the pre_mongified_id
+      def create_pre_mongified_id_index(collection_name)
+        db[collection_name].create_index([['pre_mongified_id', Mongo::ASCENDING]])
+      end
+      
       # Asks user permission to drop the database
+      # @return true or false depending on user's response
       def ask_to_drop_database
         if UI.ask("Are you sure you want to drop #{database} database?")
           drop_database

data/lib/mongify/exceptions.rb

@@ -1,20 +1,34 @@
 module Mongify
+  # Base Mongify Error
+  class MongifyError < RuntimeError; end
+  
+  # Not Implemented Error from Mongify
+  class NotImplementedMongifyError < MongifyError; end
+  
   # File Not Found Exception
-  class FileNotFound < RuntimeError; end
+  class FileNotFound < MongifyError; end
   # Raise when configuration file is missing
   class ConfigurationFileNotFound < FileNotFound; end
   # Raised when Translation file is missing
   class TranslationFileNotFound < FileNotFound; end
   
   # Basic Configuration Error Exception
-  class ConfigurationError < RuntimeError; end
+  class ConfigurationError < MongifyError; end
   # Raise when a sqlConnection is required but not given
   class SqlConnectionRequired < ConfigurationError; end
+  # Raised when a SqlConfiguration is invalid?
+  class SqlConnectionInvalid < ConfigurationError; end
   # Raised when a noSqlConnection is required but not given
   class NoSqlConnectionRequired < ConfigurationError; end
+  # Raised when a NoSqlConfiguration is invalid?
+  class NoSqlConnectionInvalid < ConfigurationError; end
+  
   # Raised when a Mongify::Database::Column is expected but not given
   class DatabaseColumnExpected < ConfigurationError; end
-  
+
   # Raised when application has no root folder set
-  class RootMissing < RuntimeError; end
+  class RootMissing < MongifyError; end
+  
+  # Raised when an invalid option is passed via CLI
+  class InvalidOption < MongifyError; end
 end

data/lib/mongify/progressbar.rb

@@ -0,0 +1,270 @@
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+# This has been modified by 
+#   Andrew Kalek
+#   Anlek Consulting
+#   http://anlek.com
+
+module Mongify
+  # Progress bar used to display results
+  class ProgressBar
+    #Progress bar version
+    VERSION = "0.9.1"
+
+    def initialize (title, total, out = STDERR)
+      @title = title
+      @total = total
+      @out = out
+      @terminal_width = 80
+      @bar_mark = "o"
+      @current = 0
+      @previous = 0
+      @finished_p = false
+      @start_time = Time.now
+      @previous_time = @start_time
+      @title_width = 35
+      @format = "%-#{@title_width}s %s %3d%% %s %s"
+      @format_arguments = [:title, :count, :percentage, :bar, :stat]
+      clear
+      show
+    end
+    attr_reader   :title
+    attr_reader   :current
+    attr_reader   :total
+    attr_accessor :start_time
+
+    #######
+    private
+    #######
+
+    # Formatting for the actual bar
+    def fmt_bar
+      bar_width = do_percentage * @terminal_width / 100
+      sprintf("|%s%s|", 
+              @bar_mark * bar_width, 
+              " " *  (@terminal_width - bar_width))
+    end
+
+    # Formatting for the percentage
+    def fmt_percentage
+      do_percentage
+    end
+
+    # Formatting for the stat (time left or time taken to complete)
+    def fmt_stat
+      if @finished_p then elapsed else eta end
+    end
+
+    # Formatting for file transfer
+    def fmt_stat_for_file_transfer
+      if @finished_p then 
+        sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+      else 
+        sprintf("%s %s %s", bytes, transfer_rate, eta)
+      end
+    end
+
+    # Formatting for title
+    def fmt_title
+      @title[0,(@title_width - 1)] + ":"
+    end
+    
+    # Formatting for count (x/y)
+    def fmt_count
+      sprintf('%15s', "(#{@current}/#{@total})")
+    end
+
+    # Converts bytes to kb, mb or gb
+    def convert_bytes (bytes)
+      if bytes < 1024
+        sprintf("%6dB", bytes)
+      elsif bytes < 1024 * 1000 # 1000kb
+        sprintf("%5.1fKB", bytes.to_f / 1024)
+      elsif bytes < 1024 * 1024 * 1000  # 1000mb
+        sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+      else
+        sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+      end
+    end
+
+    # Returns the transfer rate
+    # works only with file transfer
+    def transfer_rate
+      bytes_per_second = @current.to_f / (Time.now - @start_time)
+      sprintf("%s/s", convert_bytes(bytes_per_second))
+    end
+
+    # Gets current byte count
+    def bytes
+      convert_bytes(@current)
+    end
+    
+    # Gets formatting for time
+    def format_time (t)
+      t = t.to_i
+      sec = t % 60
+      min  = (t / 60) % 60
+      hour = t / 3600
+      sprintf("%02d:%02d:%02d", hour, min, sec);
+    end
+
+    # ETA stands for Estimated Time of Arrival.
+    def eta
+      if @current == 0
+        "ETA:  --:--:--"
+      else
+        elapsed = Time.now - @start_time
+        eta = elapsed * @total / @current - elapsed;
+        sprintf("ETA:  %s", format_time(eta))
+      end
+    end
+    
+    # Returns elapsed time
+    def elapsed
+      elapsed = Time.now - @start_time
+      sprintf("Time: %s", format_time(elapsed))
+    end
+  
+    # Returns end of line
+    # @return [String] "\n" or "\r"
+    def eol
+      if @finished_p then "\n" else "\r" end
+    end
+    
+    # Calculates percentage
+    # @return [Number] the percentage
+    def do_percentage
+      if @total.zero?
+        100
+      else
+        @current  * 100 / @total
+      end
+    end
+    
+    # Gets the width of the terminal window
+    def get_width
+      UI.terminal_helper.output_cols
+    end
+
+    # Draws the bar
+    def show
+      arguments = @format_arguments.map {|method| 
+        method = sprintf("fmt_%s", method)
+        send(method)
+      }
+      line = sprintf(@format, *arguments)
+
+      width = get_width
+      if line.length == width - 1 
+        @out.print(line + eol)
+        @out.flush
+      elsif line.length >= width
+        @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+        if @terminal_width == 0 then @out.print(line + eol) else show end
+      else # line.length < width - 1
+        @terminal_width += width - line.length + 1
+        show
+      end
+      @previous_time = Time.now
+    end
+
+    # Checks if it's needed, shows if it's so
+    def show_if_needed
+      if @total.zero?
+        cur_percentage = 100
+        prev_percentage = 0
+      else
+        cur_percentage  = (@current  * 100 / @total).to_i
+        prev_percentage = (@previous * 100 / @total).to_i
+      end
+
+      # Use "!=" instead of ">" to support negative changes
+      if cur_percentage != prev_percentage || 
+          Time.now - @previous_time >= 1 || @finished_p
+        show
+      end
+    end
+
+    public
+    # Clear's line
+    def clear
+      @out.print "\r"
+      @out.print(" " * (get_width - 1))
+      @out.print "\r"
+    end
+
+    # Marks finished
+    def finish
+      @current = @total
+      @finished_p = true
+      show
+    end
+    
+    # Returns if the bar is finished
+    def finished?
+      @finished_p
+    end
+
+    # Sets bar to file trasfer mode
+    def file_transfer_mode
+      @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+    end
+
+    # Allows format to be re/defined
+    def format= (format)
+      @format = format
+    end
+
+    # Allows to change the arguments of items that are shown
+    def format_arguments= (arguments)
+      @format_arguments = arguments
+    end
+
+    # halts drawing of bar
+    def halt
+      @finished_p = true
+      show
+    end
+
+    # Incremets bar
+    # @return [Integer] current bar frame
+    def inc (step = 1)
+      @current += step
+      @current = @total if @current > @total
+      show_if_needed
+      @previous = @current
+    end
+
+    # Allows you to set bar frame
+    # @return [Integer] current bar frame
+    def set (count)
+      if count < 0 || count > @total
+        raise "invalid count: #{count} (total: #{@total})"
+      end
+      @current = count
+      show_if_needed
+      @previous = @current
+    end
+
+    # Returns string representation of this object
+    def inspect
+      "#<ProgressBar:#{@current}/#{@total}>"
+    end
+  end
+
+  # Same as progress bar but this counts the progress backwards from 100% to 0
+  class ReversedProgressBar < ProgressBar
+    # Calculates the percentage
+    def do_percentage
+      100 - super
+    end
+  end
+end