mongify 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fb7c986182fbc86a55503066641a0f15a567e760
-  data.tar.gz: 5644dc3f3c7eddd36abdf037526a6724221647f7
+  metadata.gz: 25663b54793dab9192d511f640820faf09c99e27
+  data.tar.gz: 6372a0edd4a9eac99b8716bc3222f9914bc7602d
 SHA512:
-  metadata.gz: b9594753d2715810d2028b0681d0204857e37c5ecc27ee5e84e3663aa1a1d14daad5b9d8e8024e9522af2883b5388e402d5c5baa43e7d9f7a1d4ad4551423f1a
-  data.tar.gz: c19b8ccf2077200c84fc9f31f61f2648fc9940aea8f3db9d42b38b1b222ad7d41625d557423cf6ff01eae483dd7a4b03e6e7bfd3e775e78be3dd61dc89ccfdcb
+  metadata.gz: a5b7dcc2a292bd15845099761b34f054f640255a4c5382157148e378e0718c135b0c5e07b1eb50047ccb1afa408f55ab9d55acd25a38b3e41e26d4045a15c52d
+  data.tar.gz: 9a0e35a33b64be04874f5f351cc8a3d8ecacb325ad1ff1cb66c2a42fa5a778e50e6663d1ee5c49f1b34a49be4ec73a45bce23295936f54299c6ccbc0a79315a1

data/.gitignore

@@ -20,7 +20,10 @@ spec/tmp/*
 .bundle/*
 
 .rvmrc
+.ruby-gemset
+.ruby-version
 
 spec/support/database.yml
 
 
+log/deprecations.log

data/.yardopts

@@ -0,0 +1,3 @@
+-
+CHANGELOG.rdoc
+LICENSE

data/CHANGELOG.rdoc

@@ -1,3 +1,8 @@
+== 1.1 / 24 March 2014
+* Sync feature add (thanks to {hammady}[https://github.com/hammady])
+* Parent unset in before_save (tanks to {nessche}[https://github.com/nessche])
+* Added batch_size (recommended by {mosheka}[https://github.com/mosheka])
+* bugs and performance improvments
 == 1.0.1 / 24 December 2013
 * Updated and locked down gems to prevent Mongify from breaking when rails gets updated.
 == 1.0.0 / 28 May 2013

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    mongify (1.0.1)
+    mongify (1.1.0)
       activerecord (~> 3.2)
       activesupport (~> 3.2)
       bson_ext (~> 1.8.2)
@@ -23,29 +23,74 @@ GEM
       i18n (~> 0.6, >= 0.6.4)
       multi_json (~> 1.0)
     arel (3.0.3)
+    binding_of_caller (0.7.2)
+      debug_inspector (>= 0.0.1)
+    bond (0.5.1)
     bson (1.8.6)
     bson_ext (1.8.6)
       bson (~> 1.8.6)
     builder (3.0.4)
+    coderay (1.1.0)
+    columnize (0.3.6)
     cucumber (1.3.10)
       builder (>= 2.1.2)
       diff-lcs (>= 1.1.3)
       gherkin (~> 2.12)
       multi_json (>= 1.7.5, < 2.0)
       multi_test (>= 0.0.2)
+    debug_inspector (0.0.2)
+    debugger (1.6.6)
+      columnize (>= 0.3.1)
+      debugger-linecache (~> 1.2.0)
+      debugger-ruby_core_source (~> 1.3.2)
+    debugger-linecache (1.2.0)
+    debugger-ruby_core_source (1.3.2)
     diff-lcs (1.2.5)
     gherkin (2.12.2)
       multi_json (~> 1.3)
     highline (1.6.20)
     i18n (0.6.9)
+    interception (0.5)
+    jist (1.5.1)
+      json
+    json (1.8.1)
     metaclass (0.0.1)
+    method_source (0.8.2)
     mocha (0.14.0)
       metaclass (~> 0.0.1)
     mongo (1.8.6)
       bson (~> 1.8.6)
     multi_json (1.8.2)
     multi_test (0.0.2)
-    mysql2 (0.2.21)
+    mysql2 (0.3.15)
+    pg (0.17.0)
+    pry (0.9.12.6)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    pry-debugger (0.2.2)
+      debugger (~> 1.3)
+      pry (~> 0.9.10)
+    pry-doc (0.6.0)
+      pry (~> 0.9)
+      yard (~> 0.8)
+    pry-docmore (0.1.1)
+      pry
+      pry-doc
+    pry-plus (1.0.0)
+      bond
+      jist
+      pry-debugger
+      pry-doc
+      pry-docmore
+      pry-rescue
+      pry-stack_explorer
+    pry-rescue (1.4.0)
+      interception (>= 0.4)
+      pry
+    pry-stack_explorer (0.4.9.1)
+      binding_of_caller (>= 0.7)
+      pry (>= 0.9.11)
     rake (10.1.1)
     rspec (2.14.1)
       rspec-core (~> 2.14.0)
@@ -55,9 +100,8 @@ GEM
     rspec-expectations (2.14.4)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.4)
+    slop (3.5.0)
     sqlite3 (1.3.8)
-    sqlite3-ruby (1.3.3)
-      sqlite3 (>= 1.3.3)
     tzinfo (0.3.38)
     watchr (0.7)
     yard (0.8.7.3)
@@ -69,9 +113,11 @@ DEPENDENCIES
   cucumber (>= 0.10)
   mocha (>= 0.9.8)
   mongify!
-  mysql2 (~> 0.2.7)
+  mysql2 (~> 0.3.1)
+  pg (>= 0.17)
+  pry-plus
   rake
   rspec (>= 2.0)
-  sqlite3-ruby (>= 1.3)
+  sqlite3 (>= 1.3)
   watchr (>= 0.6)
   yard (>= 0.5.3)

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2011-2013 Andrew Kalek, Anlek Consulting
+Copyright (c) 2010-2014 Andrew Kalek, Anlek Consulting
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.rdoc

@@ -24,16 +24,17 @@ In order for Mongify to do its job, it needs to know where your databases are lo
 Building a config file is as simple as this:
 
   sql_connection do
-    adapter   "mysql"
-    host      "localhost"
-    username  "root"
-    password  "passw0rd"
-    database  "my_database"
+    adapter     "mysql"
+    host        "localhost"
+    username    "root"
+    password    "passw0rd"
+    database    "my_database"
+    batch_size  10000           # This is defaulted to 10000 but in case you want to make that smaller (on lower RAM machines)
   end
 
   mongodb_connection do
-    host      "localhost"
-    database  "my_database"
+    host        "localhost"
+    database    "my_database"
   end
 
 You can check your configuration by running
@@ -114,6 +115,7 @@ Save the file as <tt>"translation_file.rb"</tt> and run the command:
   Commands:
     "check" or "ck"           >> Checks connection for sql and no_sql databases [configuration_file]
     "process" or "pr"         >> Takes a translation and process it to mongodb [configuration_file, translation_file]
+    "sync" or "sy"            >> Takes a translation and process it to mongodb, only syncs (insert/update) new or updated records based on the updated_at column [configuration_file, translation_file]
     "translation" or "tr"     >> Outputs a translation file from a sql connection [configuration_file]
 
   Examples:
@@ -122,6 +124,7 @@ Save the file as <tt>"translation_file.rb"</tt> and run the command:
     mongify tr database.config
     mongify check database.config
     mongify process database.config database_translation.rb
+    mongify sync database.config database_translation.rb
 
   Common options:
       -h, --help                       Show this message
@@ -185,9 +188,13 @@ Table Options are as follow:
   end
 
   table "preferences", :embed_in => "users" do              # As of version 0.2, embedded tables with a before_save will take an
-    before_save do |pref_row, user_row|                     # extra argument which is the parent row of the embedded table.
+    before_save do |pref_row, user_row, unset_user_row|                     # extra argument which is the parent row of the embedded table.
       user_row.email_me = pref_row.delete('email_me')       # This gives you the ability to move things from an embedded table row
-    end                                                     # to the parent row.
+                                                            # to the parent row.
+      if pref_row['one_name']
+        unset_user_row['last_name'] = true                  # This will delete/unset the last name for the parent row
+      end
+    end
   end
 
 More documentation can be found at {Mongify::Database::Table}
@@ -247,9 +254,14 @@ More documentation can be found at {Mongify::Database::Column}
 
 == Notes
 
-Mongify has been tested on Ruby 1.8.7-p330 and Ruby 1.9.2-p136
+Mongify has been tested on Ruby 1.8.7-p330, 1.9.2-p136, 1.9.3-p327, 2.0.0-p353, 2.1.1
+
 If you have any issues, please feel free to report them here: {issue tracker}[https://github.com/anlek/mongify/issues]
 
+== Sync Note
+
+If you want to continually sync your sql database with MongoDB using Mongify, you MUST run the sync feature from the start. Using the process command will not permit future syncing. The sync feature leaves the `pre_mongify_id` in your documents as well as it creates it's own collection called `__mongify_sync_helper__` to keep dates and times of your last sync.
+
 == Additional Tools
 
 As of May 2013, we've released an add-on tool to Mongify called {Mongify-Mongoid}[https://github.com/anlek/mongify-mongoid] which lets you use the translation.rb file to generate Mongoid models. It generates the model fields, relations (most of the way) and timestamps. This should save you some time from having to re-type all the models by hand.
@@ -272,7 +284,8 @@ You just need bundler >= 1.0.8
   gem install bundler
   bundle install
   copy over spec/support/database.example to spec/support/database.yml and fill in required info
-  rake test:mysql:setup
+  rake test:setup:mysql
+  rake test:setup:postgresql
   rake test
 
 == Special Thanks
@@ -292,6 +305,11 @@ Reach me at:
 * Twitter: {@anlek}[http://www.twitter.com/anlek]
 * Email: andrew.kalek@anlek[dot]com
 
+The sync functionality was made by Hossam Hammady from Qatar Computing Research Institute
+
+Reach me at:
+* Twitter: {@hammady}[http://www.twitter.com/hammady]
+* Email: hhammady@qf[dot]org[dot]qa
 
 == License
 

data/Rakefile

@@ -18,17 +18,17 @@ end
 
 if RUBY_VERSION.to_f == 1.8
   namespace :rcov do
-    Cucumber::Rake::Task.new(:cucumber) do |t|    
+    Cucumber::Rake::Task.new(:cucumber) do |t|
       t.rcov = true
       t.rcov_opts = %w{--exclude osx\/objc,gems\/,spec\/,features\/ --aggregate coverage.data}
       t.rcov_opts << %[-o "coverage"]
     end
-   
+
     RSpec::Core::RakeTask.new(:rspec) do |t|
       t.rcov = true
       t.rcov_opts = %w{--exclude osx\/objc,gems\/,spec\/,features\/}
     end
-   
+
     desc "Run both specs and features to generate aggregated coverage"
     task :all do |t|
       rm "coverage.data" if File.exist?("coverage.data")
@@ -42,18 +42,47 @@ desc "Run rspec test"
 task :test do
   Rake::Task["test:rspec"].invoke
   if RUBY_VERSION.to_f < 1.9
-    Rake::Task["rcov:cucumber"].invoke 
+    Rake::Task["rcov:cucumber"].invoke
   else
-    Rake::Task["test:cucumber"].invoke 
+    Rake::Task["test:cucumber"].invoke
   end
 end
 namespace :test do
   RSpec::Core::RakeTask.new(:rspec)
   Cucumber::Rake::Task.new(:cucumber)
-  
-  namespace :mysql do
+
+  namespace :setup do
+    desc "Setup a postgresql database based on the spec/support/database.yml settings (under postgresql)"
+    task :postgresql do
+      require './spec/support/config_reader'
+      require 'active_record'
+      ::CONNECTION_CONFIG = ConfigReader.new('spec/support/database.yml')
+      ActiveRecord::Base.establish_connection(CONNECTION_CONFIG.postgresql)
+      conn = ActiveRecord::Base.connection
+      conn.create_table(:users, :force => true) do |t|
+        t.string :first_name, :last_name
+        t.timestamps
+      end
+
+      conn.create_table(:posts, :force => true) do |t|
+        t.string :title
+        t.integer :owner_id
+        t.text :body
+        t.datetime :published_at
+        t.timestamps
+      end
+
+      conn.create_table(:comments, :force => true) do |t|
+        t.text :body
+        t.integer :post_id
+        t.integer :user_id
+        t.timestamps
+      end
+      puts "Finished"
+    end
+
     desc "Setup a mysql database based on the spec/support/database.yml settings"
-    task :setup do
+    task :mysql do
       require './spec/support/config_reader'
       require 'active_record'
       ::CONNECTION_CONFIG = ConfigReader.new('spec/support/database.yml')
@@ -83,5 +112,4 @@ namespace :test do
   end
 end
 
-
 task :default => ['test']

data/features/options.feature

@@ -24,6 +24,7 @@ Feature: Mongify can be controlled using command-line options
       Commands:
       "check" or "ck"           >> Checks connection for sql and no_sql databases [configuration_file]
       "process" or "pr"         >> Takes a translation and process it to mongodb [configuration_file, translation_file]
+      "sync" or "sy"            >> Takes a translation and process it to mongodb, only syncs (insert/update) new or updated records based on the updated_at column [configuration_file, translation_file]
       "translation" or "tr"     >> Outputs a translation file from a sql connection [configuration_file]
 
       Examples:
@@ -31,6 +32,7 @@ Feature: Mongify can be controlled using command-line options
       mongify check database.config
       mongify translation datbase.config > database_translation.rb
       mongify process database.config database_translation.rb
+      mongify sync database.config database_translation.rb
 
       See http://github.com/anlek/mongify for more details
 

data/features/print.feature

@@ -2,7 +2,7 @@ Feature: Database Translation Output
   In order to translate sql database to a no-sql database
   As a developer / database admin
   Needs to be able to generate a translation file
-  
+
   Scenario: Translation Output request
     Given a database exists
     When I run mongify translation spec/files/base_configuration.rb

data/features/process.feature

@@ -14,7 +14,7 @@ Feature: Processing a translation
   And there should be 0 comments in mongodb
   And the post with title "First Post" should have 1 comment
   And the post with title "Second Post" should have 2 comments
-  
+
   Scenario: Processing while modifying embedding parent.
   Given a database exists
   And a blank mongodb
@@ -23,3 +23,12 @@ Feature: Processing a translation
   And there should be 3 users in mongodb
   And the first user's notify_by_email attribute should be true
   And the third user's notify_by_email attribute should be false
+
+  Scenario: Processing while deleting fields from embedding parent
+  Given a database exists
+  And a blank mongodb
+  When I run mongify process spec/files/base_configuration.rb spec/files/deleting_fields_from_embedding_parent_translation.rb
+  Then it succeeds
+  And there should be 3 teams in mongodb
+  And the first team's phone attribute should not be present
+  And the third team's coach's phone attribute should be +1112223334

data/features/step_definitions/process_steps.rb

@@ -1,4 +1,14 @@
-Then /^the (first|second|third) (.+)'s (.+) attribute should be (.+)$/ do |collection_place, collection_name, field, value|
+Then /^the (first|second|third) ([^\s]+)'s ([^\s]+) attribute should be (.+)$/ do |collection_place, collection_name, field, value|
   DatabaseGenerator.mongo_connection.db[collection_name.pluralize].
                    find.to_a.send(collection_place.to_sym)[field].to_s.should == value.to_s
+end
+
+Then /^the (first|second|third) ([^\s]+)'s ([^\s]+) attribute should( not)? be present$/ do |collection_place, collection_name, field, negate|
+  DatabaseGenerator.mongo_connection.db[collection_name.pluralize].
+                    find.to_a.send(collection_place.to_sym).has_key?(field).should (negate ? be_false : be_true)
+end
+
+Then /^the (first|second|third) ([^\s]+)'s ([^\s]+)'s ([^\s]+) attribute should be (.+)$/ do |collection_place, collection_name, embedded_name, field, value|
+  DatabaseGenerator.mongo_connection.db[collection_name.pluralize].
+                    find.to_a.send(collection_place.to_sym)[embedded_name][field].to_s.should == value.to_s
 end

data/features/support/env.rb

@@ -23,7 +23,7 @@ module MongifyWorld
     @last_exit_status = $?.exitstatus
     @last_stderr = IO.read(stderr_file.path)
   end
-   
+
   # Executes mongify command with arguments
   def mongify(args)
     run("ruby -Ilib -rubygems bin/mongify #{args}")

data/lib/mongify/cli/application.rb

@@ -7,12 +7,12 @@ module Mongify
     # command line.
     #
     class Application
-      
+
       # Successful execution exit code
       STATUS_SUCCESS = 0
-      # Failed execution exit code 
+      # Failed execution exit code
       STATUS_ERROR   = 1
-      
+
       def initialize(arguments=[], stdin=$stdin, stdout=$stdout)
         arguments = ['-h'] if arguments.empty?
         @options = Options.new(arguments)
@@ -20,7 +20,7 @@ module Mongify
         Mongify::Configuration.in_stream = stdin
         Mongify::Configuration.out_stream = stdout
       end
-      
+
       # Runs the application
       def execute!
         begin
@@ -34,17 +34,17 @@ module Mongify
           raise error
         end
       end
-      
+
       # Sends output to the UI
       def output(message)
         UI.puts(message)
       end
-      
+
       # Sets status code as successful
       def report_success
         @status = STATUS_SUCCESS
       end
-      
+
       # Sets status code as failure (or error)
       def report_error
         @status = STATUS_ERROR

data/lib/mongify/cli/command/worker.rb

@@ -6,15 +6,16 @@ module Mongify
       #
       class Worker
         attr_accessor :view
-        
+
         # A hash of available commands
         # Including description, additional shortcuts to run the commands and requirements to run the command
         AVAILABLE_COMMANDS = {
                                   :check => {:commands => ['check', 'ck'], :description => "Checks connection for sql and no_sql databases", :required => [:configuration_file]},
                                   :translation => {:commands => ['translation', 'tr'], :description => "Outputs a translation file from a sql connection", :required => [:configuration_file]},
-                                  :process => {:commands => ['process', 'pr'], :description => "Takes a translation and process it to mongodb", :required => [:configuration_file, :translation_file]}
+                                  :process => {:commands => ['process', 'pr'], :description => "Takes a translation and process it to mongodb", :required => [:configuration_file, :translation_file]},
+                                  :sync => {:commands => ['sync', 'sy'], :description => "Takes a translation and process it to mongodb, only syncs (insert/update) new or updated records based on the updated_at column", :required => [:configuration_file, :translation_file]}
                                 }
-        
+
         # Prints out a nice display of the list of commands
         def self.list_commands
           [].tap do |commands|
@@ -23,7 +24,7 @@ module Mongify
             end
           end.sort
         end
-                              
+
         # Finds a command array by any of the shortcut commands
         def self.find_command(command)
           AVAILABLE_COMMANDS.each do |key, options|
@@ -31,20 +32,20 @@ module Mongify
           end
           'unknown'
         end
-        
+
         def initialize(command, config=nil, translation_file=nil, parser="")
           @command = command.to_s.downcase
           @config = config
           @translation_file = translation_file
           @parser = parser
         end
-        
+
         #Executes the worked based on a given command
         def execute(view)
           self.view = view
-          
+
           current_command, command_options = find_command
-          
+
           if command_options
             #FIXME: Should parse configuration file in this action, (when I know it's required)
             raise ConfigurationFileNotFound, "Database Configuration file is missing or cannot be found" if command_options[:required] && command_options[:required].include?(:configuration_file) && @config.nil?
@@ -54,7 +55,7 @@ module Mongify
               @translation = Translation.parse(@translation_file)
             end
           end
-          
+
           case current_command
           when :translation
             check_connections
@@ -65,18 +66,21 @@ module Mongify
           when :process
             check_connections
             @translation.process(@config.sql_connection, @config.no_sql_connection)
+          when :sync
+            check_connections
+            @translation.sync(@config.sql_connection, @config.no_sql_connection)
           else
             view.output("Unknown action #{@command}\n\n#{@parser}")
             return view.report_error
           end
           view.report_success
         end
-        
+
         # Passes find command to parent class
         def find_command(command=@command)
           self.class.find_command(command)
         end
-        
+
         #######
         private
         #######
@@ -85,15 +89,15 @@ module Mongify
         def check_connections
           check_sql_connection && check_nosql_connection
         end
-        
+
         # Checks sql connection if it's valid and has_connection?
         def check_sql_connection
           @config.sql_connection.valid? && @config.sql_connection.has_connection?
         end
-        
+
         # Checks no sql connection if it's valid and has_connection?
         def check_nosql_connection
-          @config.no_sql_connection.valid? && @config.no_sql_connection.has_connection?        
+          @config.no_sql_connection.valid? && @config.no_sql_connection.has_connection?
         end
       end
     end