mongify 1.0.1 → 1.1.0

data/lib/mongify/cli/options.rb

@@ -28,6 +28,7 @@ Examples:
 #{progname} check database.config
 #{progname} translation datbase.config > database_translation.rb
 #{progname} process database.config database_translation.rb
+#{progname} sync database.config database_translation.rb
 
 See http://github.com/anlek/mongify for more details
 
@@ -85,4 +86,4 @@ EOB
       end
     end
   end
-end
+end

data/lib/mongify/configuration.rb

@@ -6,7 +6,7 @@ module Mongify
   class Configuration
     class << self
       attr_accessor :in_stream, :out_stream
-      
+
       # Parses a external configuration file and evaluates it and returns a instence of a configuration class
       def parse(file_name)
         raise Mongify::ConfigurationFileNotFound, "File #{file_name} is missing" unless File.exists?(file_name)
@@ -16,7 +16,7 @@ module Mongify
       end
 
     end #self
-    
+
     # Returns a no_sql_connection which is bound to a mongodb adapter
     # or builds a new no_sql_connection if block is given
     def mongodb_connection(options={}, &block)
@@ -25,7 +25,7 @@ module Mongify
       options['adapter'] ||= 'mongodb'
       @mongodb_connection = no_sql_connection(options, &block)
     end
-    
+
     # Returns a sql_connection
     # If a block is given, it will be executed on the connection
     # For more information, see {Mongify::Database::SqlConnection}
@@ -34,7 +34,7 @@ module Mongify
       @sql_connection.instance_exec(&block) if block_given?
       @sql_connection
     end
-    
+
     # Returns a sql_connection
     # If a block is given, it will be executed on the connection
     # For more information, see {Mongify::Database::NoSqlConnection}
@@ -43,6 +43,6 @@ module Mongify
       @no_sql_connection.instance_exec(&block) if block_given?
       @no_sql_connection
     end
-    
+
   end
 end

data/lib/mongify/database/base_connection.rb

@@ -8,10 +8,10 @@ module Mongify
       # List of required fields to make a valid base connection
       REQUIRED_FIELDS = %w{host}
       # List of all the available fields to make up a connection
-      AVAILABLE_FIELDS = %w{adapter host username password database socket port encoding}
+      AVAILABLE_FIELDS = %w{adapter host username password database socket port encoding batch_size}
       # List of all fields that should be forced to a string
       STRING_FIELDS = %w{adapter}
-      
+
       def initialize(options=nil)
         if options
           options.stringify_keys!
@@ -20,7 +20,7 @@ module Mongify
           end
         end
       end
-      
+
       # Returns all settings as a hash, this is used mainly in building ActiveRecord::Base.establish_connection
       def to_hash
         hash = {}
@@ -53,15 +53,15 @@ module Mongify
 
 
       # Returns true if we are trying to respond_to AVAILABLE_FIELDS functions
-      def respond_to?(method, *args) 
+      def respond_to?(method, *args)
         return true if AVAILABLE_FIELDS.include?(method.to_s)
         super(method)
       end
-      
+
       # Building set and/or return functions for AVAILABLE_FIELDS.
       # STRING_FIELDS are forced into string.
       # Example:
-      # 
+      #
       #   def host(value=nil)
       #     @host = value.to_s unless value.nil?
       #     @host

data/lib/mongify/database/column.rb

@@ -4,17 +4,17 @@ module Mongify
     #
     # A column that is used to access sql data and transform it into the no sql database
     #
-    # 
+    #
     # ==== Structure
-    # 
+    #
     # Structure for defining a column is as follows:
     #   column "name", :type, {options}
     # <em>Columns with no type given will be set to <tt>:string</tt></em>
-    # 
+    #
     # ==== Notes
     # Leaving a column out when defining a table will result in a copy of the information (as a string).
     # ==== Types
-    # 
+    #
     # Types of columns are supported:
     #   :key                  # Columns that are primary keys need to be marked as :key type. You can provide an :as if your :key is not an integer column
     #   :integer              # Will be converted to a integer
@@ -28,42 +28,42 @@ module Mongify
     #   :time                 # Will be converted to a Time format (the date portion of the Time object will be 2000-01-01)
     #   :binary               # Will be converted to a string
     #   :boolean              # Will be converted to a true or false values
-    # 
+    #
     # ==== Options
-    # 
+    #
     #   column "post_id", :integer, :references => :posts   # Referenced columns need to be marked as such, this will mean that they will be updated
     #                                                       # with the new BSON::ObjectID.
     # <b>NOTE: if you rename the table 'posts', you should set the :references to the new name</b>
-    # 
+    #
     #   column "name", :string, :ignore => true             # Ignoring a column will make the column NOT copy over to the new database
-    # 
+    #
     #   column "surname", :string, :rename_to => 'last_name'# Rename_to allows you to rename the column
     #
     #   column "post_id", :integer, :auto_detect => true    # Will run auto detect and make this column a :references => 'posts', :on => 'post_id' for you
     #                                                       # More used when reading a sql database, NOT recommended for use during processing of translation
-    # 
+    #
     # <em>For decimal columns you can specify a few options:</em>
     #   column "total",                                     # This is a default conversion setting.
     #          :decimal,
-    #          :as => 'string'                              
-    #   
+    #          :as => 'string'
+    #
     #   column "total",                                     # You can specify to convert your decimal to integer
     #           :decimal,                                   # specifying scale will define how many decimal places to keep
     #           :as => 'integer',                           # Example: :scale => 2 will convert 123.4567 to 12346 in to mongodb
     #           :scale => 2
     # ==== Decimal Storage
-    # 
+    #
     # Unfortunately MongoDB Ruby Drivers doesn't support BigDecimal, so to ensure all data is stored correctly (without losing information)
     # I've chosen to store as String, however you can overwrite this functionality in one of two ways:
     # <em>The reason you would want to do this, is to make this searchable via a query.</em>
-    # 
+    #
     # <b>1) You can specify :as => 'integer', :scale => 2</b>
     #   column "total", :decimal, :as => 'integer', :scale => 2
-    # 
+    #
     #   #It would take a value of 123.456 and store it as an integer of value 12346
-    # 
+    #
     # <b>2) You can specify your own custom conversion by doing a {Mongify::Database::Table#before_save}
-    # 
+    #
     # Example:
     #   table "invoice" do
     #     column "name", :string
@@ -72,16 +72,16 @@ module Mongify
     #       row.total = (BigDecimal.new(row.total) * 1000).round
     #     end
     #   end
-    # 
+    #
     # This would take 123.456789 in the total column and convert it to an interger of value 123457 (and in your app you can convert it back to a decimal)
-    # 
+    #
     # *REMEMBER* there is a limit on how big of an integer you can store in BSON/MongoDB (http://bsonspec.org/#/specification)
     class Column
       attr_reader :sql_name, :type, :options
-      
+
       #List of available options for a column
       AVAILABLE_OPTIONS = ['references', 'ignore', 'rename_to', 'as', 'scale']
-      
+
       # Auto detects if a column is an :key column or is a reference column
       def self.auto_detect(column)
         case column.sql_name.downcase
@@ -92,7 +92,7 @@ module Mongify
           column.references = $1.to_s.pluralize unless column.referenced?
         end
       end
-      
+
       def initialize(sql_name, type=:string, options={})
         @sql_name = sql_name
         options, type = type, nil if type.is_a?(Hash)
@@ -100,21 +100,21 @@ module Mongify
         @options = options.stringify_keys
         @auto_detect = @options.delete('auto_detect')
         run_auto_detect!
-        
+
         self
       end
-      
+
       # Allows you to set a type
       def type=(value=:string)
         value = :string if value.nil?
         @type = value.is_a?(Symbol) ? value : value.to_sym
       end
-      
+
       # Returns the no_sql record name
       def name
         @name ||= rename_to || sql_name
       end
-      
+
       # Returns a translated hash from a given value
       # Example:
       #   @column = Column.new("surname", :string, :rename_to => 'last_name')
@@ -128,12 +128,12 @@ module Mongify
             {"#{self.name}" => type_cast(value)}
         end
       end
-      
+
       # Returns a string representation of the column as it would show in a translation file.
       # Mainly used during print out of translation file
       def to_print
         "column \"#{sql_name}\", :#{type}".tap do |output|
-          output_options = options.map do |k, v| 
+          output_options = options.map do |k, v|
             next if v.nil?
             ":#{k} => #{v.is_a?(Symbol) ? ":#{v}" : %Q["#{v}"] }"
           end.compact
@@ -141,7 +141,7 @@ module Mongify
         end
       end
       alias :to_s :to_print
-      
+
       # Sets up a accessor method for an option
       #
       #   def rename_to=(value)
@@ -172,32 +172,32 @@ module Mongify
           super(meth, *args, &blk)
         end
       end
-      
+
       # Returns true if the column is a reference column
       def referenced?
         !self.options['references'].nil?
       end
-      
+
       # Returns true if column is being renamed
       def renamed?
         self.name != self.sql_name
       end
-      
+
       # Returns true if column is a :key type column
       def key?
         self.type == :key
       end
-      
+
       # Returns true if column should be auto_detected (passed via options)
       def auto_detect?
         !!@auto_detect
       end
-      
+
       # Returns true if column is ignored
       def ignored?
         !!self.ignore
       end
-      
+
       # Used when trying to figure out how to convert a decimal value
       # @return [String] passed option['as'] or defaults to 'string'
       def as
@@ -213,7 +213,7 @@ module Mongify
       def as_integer?
         self.as == :integer
       end
-      
+
       # Get the scale option for decimal to integer conversion
       #   column 'total', :decimal, :as => 'integer', :scale => 3
       # @return [integer] passed option['scale'] or 0
@@ -229,7 +229,7 @@ module Mongify
       #######
       private
       #######
-      
+
       # Casts the value to a given type
       def type_cast(value)
         return nil if value.nil?
@@ -243,7 +243,7 @@ module Mongify
             value = ActiveRecord::ConnectionAdapters::Column.value_to_decimal(value)
             if as_integer?
               (value * (10 ** self.scale)).round.to_i
-            else     
+            else
               value.to_s
             end
           when :datetime  then ActiveRecord::ConnectionAdapters::Column.string_to_time(value)
@@ -256,14 +256,14 @@ module Mongify
         end
       end
 
-      
+
 
       # runs auto detect (see {Mongify::Database::Column.auto_detect})
       def run_auto_detect!
         self.class.auto_detect(self) if auto_detect?
       end
-      
-      
+
+
     end
   end
 end

data/lib/mongify/database/data_row.rb

@@ -6,28 +6,28 @@ module Mongify
         hash = {} if hash.nil?
         @hash = hash.dup.stringify_keys!
       end
-      
+
       # See if a given key is included
       def include?(key)
         @hash.has_key?(key)
       end
-      
+
       # Deletes a given key from the object
       def delete(key)
         @hash.delete(key)
       end
-      
+
       # Returns a list of available keys
       def keys
         @hash.keys
       end
-      
+
       # Outputs an hash
       # This is used to write into the no sql database
       def to_hash
         @hash
       end
-      
+
       # Used to manually read an attribute
       def read_attribute(key)
         @hash[key.to_s]
@@ -36,18 +36,18 @@ module Mongify
       def write_attribute(key, value)
         @hash[key.to_s] = value
       end
-      
+
       # Passes Inspect onto the internal hash
       def inspect
         @hash.inspect
       end
-      
+
       # Updated respond_to to return true if it's a key the hash
       def respond_to?(method)
         return true if @hash.has_key?(method.gsub('=', ''))
         super(method)
       end
-      
+
       # Added the ability to read and write attributes in the hash
       def method_missing(meth, *args, &blk)
         match = meth.to_s.match(/^([a-zA-Z\_]+)(=|$)$/)
@@ -62,7 +62,7 @@ module Mongify
           super(meth, *args, &blk)
         end
       end
-      
+
     end
   end
 end

data/lib/mongify/database/no_sql_connection.rb

@@ -3,15 +3,15 @@ module Mongify
   module Database
     #
     # No Sql Connection configuration
-    # 
+    #
     # Basic format should look something like this:
-    # 
+    #
     #   no_sql_connection {options} do
     #     adapter   "mongodb"
     #     host      "localhost"
     #     database  "my_database"
     #   end
-    # 
+    #
     # Possible attributes:
     #   adapter
     #   host
@@ -19,106 +19,132 @@ module Mongify
     #   username
     #   password
     #   port
-    # 
+    #
     # Options:
-    #   :force => true       # This will force a database drop before processing  
+    #   :force => true       # This will force a database drop before processing
     # <em>You're also able to set attributes via the options</em>
     #
     class NoSqlConnection < Mongify::Database::BaseConnection
       include Mongo
-      
-      
+
+
       #Required fields for a no sql connection
-      REQUIRED_FIELDS = %w{host database}  
-      
+      REQUIRED_FIELDS = %w{host database}
+
       def initialize(options={})
         super options
         @options = options
-        adapter 'mongodb' if adapter.nil? || adapter == 'mongo'
+        adapter 'mongodb' if adapter.nil? || adapter.downcase == "mongo"
       end
-      
+
       # Sets and/or returns a adapter
       # It takes care of renaming adapter('mongo') to 'mongodb'
       def adapter(name=nil)
-        name = 'mongodb' if name && name.to_s.downcase == 'mongo'
         super(name)
       end
-      
+
       # Returns a connection string that can be used to build a Mongo Connection
       # (Currently this isn't used due to some issue early on in development)
       def connection_string
         "#{@adapter}://#{@host}#{":#{@port}" if @port}"
       end
-      
-      # Returns true or false depending if the given attributes are present and valid to make up a 
+
+      # Returns true or false depending if the given attributes are present and valid to make up a
       # connection to a mongo server
       def valid?
         super && @database.present?
       end
-      
+
       # Returns true if :force was set to true
       # This will force a drop of the database upon connection
       def forced?
         !!@options['force']
       end
-      
+
       # Sets up a connection to the database
       def setup_connection_adapter
         connection = Connection.new(host, port)
         connection.add_auth(database, username, password) if username && password
         connection
       end
-      
+
       # Returns a mongo connection
       # NOTE: If forced? is true, the first time a connection is made, it will ask to drop the
-      # database before continuing 
+      # database before continuing
       def connection
         return @connection if @connection
         @connection = setup_connection_adapter
         @connection
       end
-      
+
       # Returns true or false depending if we have a connection to a mongo server
       def has_connection?
         connection.connected?
       end
-      
+
       # Returns the database from the connection
       def db
         @db ||= connection[database]
       end
-      
+
       # Returns a hash of all the rows from the database of a given collection
       def select_rows(collection)
         db[collection].find
       end
-      
+
+      def select_by_query(collection, query)
+        db[collection].find(query)
+      end
+
       # Inserts into the collection a given row
       def insert_into(colleciton_name, row)
         db[colleciton_name].insert(row)
       end
-      
+
       # Updates a collection item with a given ID with the given attributes
       def update(colleciton_name, id, attributes)
         db[colleciton_name].update({"_id" => id}, attributes)
       end
-      
+
+      # Upserts into the collection a given row
+      def upsert(collection_name, row)
+        # We can't use the save method of the Mongo collection
+        # The reason is that it detects duplicates using _id
+        # but we should detect it using pre_mongified_id instead
+        # because in the case of sync, same rows are identified by their original sql ids
+        #
+        # db[collection_name].save(row)
+
+        if row.has_key?(:pre_mongified_id) || row.has_key?('pre_mongified_id')
+          id = row[:pre_mongified_id] || row['pre_mongified_id']
+          duplicate = find_one(collection_name, {"pre_mongified_id" => id})
+          if duplicate
+            update(collection_name, duplicate[:_id] || duplicate["_id"], row)
+          else
+            insert_into(collection_name, row)
+          end
+        else
+          # no pre_mongified_id, fallback to the upsert method of Mongo
+          db[collection_name].save(row)
+        end
+      end
+
       # Finds one item from a collection with the given query
       def find_one(collection_name, query)
         db[collection_name].find_one(query)
       end
-      
+
       # Returns a row of a item from a given collection with a given pre_mongified_id
       def get_id_using_pre_mongified_id(colleciton_name, pre_mongified_id)
         db[colleciton_name].find_one('pre_mongified_id' => pre_mongified_id).try(:[], '_id')
       end
-      
+
       # Removes pre_mongified_id from all records in a given collection
       def remove_pre_mongified_ids(collection_name)
         drop_mongified_index(collection_name)
         db[collection_name].update({}, { '$unset' => { 'pre_mongified_id' => 1} }, :multi => true)
       end
-      
+
       # Removes pre_mongified_id from collection
       # @param [String] collection_name name of collection to remove the index from
       # @return True if successful
@@ -126,13 +152,13 @@ module Mongify
       def drop_mongified_index(collection_name)
         db[collection_name].drop_index('pre_mongified_id_1') if db[collection_name].index_information.keys.include?("pre_mongified_id_1")
       end
-      
+
       # Creates a pre_mongified_id index to ensure
       # speedy lookup for collections via the pre_mongified_id
       def create_pre_mongified_id_index(collection_name)
         db[collection_name].create_index([['pre_mongified_id', Mongo::ASCENDING]])
       end
-      
+
       # Asks user permission to drop the database
       # @return true or false depending on user's response
       def ask_to_drop_database
@@ -140,18 +166,18 @@ module Mongify
           drop_database
         end
       end
-      
+
       #######
       private
       #######
-      
+
       # Drops the mongodb database
       def drop_database
         connection.drop_database(database)
       end
-      
 
-      
+
+
     end
   end
-end
+end