mongar 0.0.6 → 0.0.7

data/README.rdoc

@@ -1,89 +1,157 @@
 = mongar
 
-!! This is still fresh off the oven and hasn't been thoroughly tested yet in production. !!
+Replicate data from MySQL, PostgreSQL, SQLite, etc to MongoDB!
 
-Ruby GEM to facilitate replicating data from ActiveRecord (or any data represented by a set of Ruby classes) to MongoDB.
+Mongar will replicate data from ActiveRecord (or any data represented by a Ruby class) to MongoDB.
+For example, if you have MySQL data represented by ActiveRecord, you can easily replicate it to
+MongoDB.
 
 == Adding to a rails project
 
+For now, you'll have to manually add Mongar to your Rails project.  We'll work on a Railtie for Rails 3 soon.
+
 Put your Mongar config into config/mongar.rb (See Example Config below)
 
-Put the following code into script/mongar.rb
+Put the following code into your Rakefile
 
   require 'mongar'
-  mongar_config = File.join(Rails.root, 'config', 'mongar.rb')
-  mongar = eval(File.read(mongar_config))
-  mongar.run
 
-Run it with rails runner (rails 3.x) or script/runner (rails 2.x) script/mongar.rb
+  namespace :mongar do
+    desc "Run mongar replication"
+    task :run => :environment do
+      mongar_config = File.join(Rails.root, 'config', 'mongar.rb')
+      mongar = eval(File.read(mongar_config))
+      mongar.run
+    end
+  end
+
+Run it with 
+
+  rake mongar:run
+
+== Assumptions
+
+The default configuration assumes that you are using ActiveRecord and that are you using a plugin
+similar to acts_as_paranoid that never deletes records from the database but only marks them as
+deleted with a deleted_at column.
+
+You can customize the finder methods that Mongar uses to accommodate any data model or your own
+custom scopes.
+
+== Minimal example config
+
+Given an ActiveRecord model call Domain with two attributes name and client_id:
+
+  Mongar.configure do
+    mongo :default do
+      database 'mydb'
+    end
+    
+    replicate Domain do
+      column :name do
+        primary_index
+      end
+      
+      column :client_id
+    end
+  end
 
-== Example Config
+== Full example Config
 
   Mongar.configure do
+    # Currently we only log to STDOUT, we have plans to use a
+    # plans to use a configurable logger.
+    # Valid log levels are :info and :debug
     log_level :debug
     
     mongo :default do
       database 'mydb'
-      user 'mongouser'
-      password 'password'
-      host '127.0.0.1'
-      port 27017
     end
   
     mongo :otherdb do
-      database 'mydb'
+      database 'myotherdb'
       user 'mongouser'
       password 'password'
-    
-      status_collection :statuses
+      host '127.0.0.1'
+      port 27017
+      
+      # By default Mongar uses the 'statuses' collection to
+      # keep track of the last replicated times, you can specify
+      # a custom collection name
+      status_collection :stati
     end
-  
-    replicate Domain => 'domains' do
-      use_mongodb :otherdb
     
-      full_refresh :every => 60.minutes
-    
-      column :uri do
+    # Minimal replica config to replicate the Domain model
+    # to the 'domains' mongo collection on the mongo db defined
+    # in the :default config above
+  
+    replicate Domain do
+      column :name do
         primary_index
-        transform :downcase
       end
-    
-      column :allow_anyone_to_anyone_policy
+      
+      column :client_id
     end
   
-    replicate Client do
-      db_time_selector do
-        current_time_on_db_server
-      end
+    # Customized replica config to replicate the Client
+    # model to the 'customers' mongo collection on the db
+    # defined in the :otherdb config above
     
-      column :id do
-        primary_index
-      end
-    
-      column :name
-
-      column :employee_count do
-        transform do |value|
-          value.nil? ? 0 : value
-        end
+    replicate Client => 'customers' do
+      # Use the other mongo database 
+      use_mongodb :otherdb
+      
+      # By default, Mongar will try to determine the time on the
+      # backend database. The supported ActiveRecord database adapters
+      # are MysqlAdapter, Mysql2Adapter, and SQLServerAdapter.
+      # You can use a custom function to determine the time on Client's
+      # backend database
+      db_time_selector do
+        # this will run Client#get_db_time
+        get_db_time
       end
-    end
-  
-    replicate EmailAddress do
+      
+      # Items are never deleted
       no_deleted_finder
+      
+      # Customize your updated record finder.  The block is 
+      # run in the source class context
       set_updated_finder do |last_replicated_date|
         find(:all, :conditions => ['something > ?', last_replicated_date])
       end
+      
+      # Custom created item finder
       set_created_finder do |last_replicated_date|
         created_scope(last_replicated_date)
       end
     
+      # Run a full refresh of all items in the database based
+      # a condition returned by the Proc
       full_refresh :if => Proc.new do |last_replicated_date|
         # class eval'ed code
         any_changes_since?(last_replicated_date)
       end
-    
-      column :address
+      
+      # You can also refresh every N seconds
+      # full_refresh :every => 3600
+      
+      # Define your columns
+      column :id do
+        # The 'primary index' column is used to find items in the mongo collection
+        primary_index
+      end
+      
+      column :tag do
+        # Run the procedure :downcase on the value of Client#tag
+        transform :downcase
+      end
+      
+      column :employee_count do
+        # Run a block transformation on the value of Client#employee_count
+        transform do |value|
+          value.nil? ? 0 : value
+        end
+      end
     end
   end
 

data/VERSION

@@ -1 +1 @@
-0.0.6
+0.0.7

data/lib/mongar/mongo/collection.rb

@@ -38,7 +38,7 @@ class Mongar::Mongo
     end
     
     def last_replicated_at=(date)
-      info "Saving #{name} last_replicated_at to #{date}"
+      info "   * Updating #{name}.last_replicated_at to #{date}"
       status_collection.update({ :collection_name => name }, 
                                { :collection_name => name, :last_replicated_at => date }, 
                                { :upsert => true })
@@ -71,13 +71,13 @@ class Mongar::Mongo
     end
     
     def mark_all_items_pending_deletion
-      info "Marking all items in #{name} for pending deletion"
+      info "   * Marking all items in #{name} for pending deletion"
       
       collection.update({ '_id' => { '$exists' => true } }, { "$set" => { :pending_deletion => true } }, { :multi => true, :safe => true })
     end
     
     def delete_all_items_pending_deletion
-      info "Deleting all items in #{name} that are pending deletion"
+      info "   * Deleting all items in #{name} that are pending deletion"
       
       collection.remove({ :pending_deletion => true }, { :safe => true })
     end

data/lib/mongar/replica.rb

@@ -30,8 +30,13 @@ class Mongar
     def run
       time = current_time_on_database_server
       
+      # Set the time back 1 second to make sure we don't miss any changes made mid-second
+      time -= 1 unless time.nil?
+      
+      info "Replicating #{source.to_s} to #{destination.name}"
+      
       if do_full_refresh?
-        info "Running full refresh on Replica #{source.to_s} to #{destination.name}"
+        info " * Full refresh"
         
         destination.mark_all_items_pending_deletion!
         
@@ -41,7 +46,7 @@ class Mongar
       else
         last_replicated_at = destination.last_replicated_at
         
-        info "Running incremental replication on Replica #{source.to_s} to #{destination.name} from #{last_replicated_at}"
+        info " * Incremental updates since #{last_replicated_at}"
         
         run_sync_for([:deleted, :created_or_updated, :updated], last_replicated_at)
       end
@@ -49,6 +54,8 @@ class Mongar
     end
     
     def run_sync_for(types, last_replicated_at)
+      info "   * Syncing #{types.join(', ')} items"
+      
       # find deleted
       find(:deleted, last_replicated_at).each do |deleted_item|
         destination.delete! source_object_to_primary_key_hash(deleted_item)

data/mongar.gemspec

@@ -4,14 +4,14 @@
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
-  s.name = "mongar"
-  s.version = "0.0.6"
+  s.name = %q{mongar}
+  s.version = "0.0.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Philippe Green"]
-  s.date = "2011-12-15"
-  s.description = "Replicates data from ActiveRecord (or other Ruby data mapping class) to MongoDB"
-  s.email = "phil@greenviewdata.com"
+  s.date = %q{2011-12-16}
+  s.description = %q{Replicates data from ActiveRecord (or other Ruby data mapping class) to MongoDB}
+  s.email = %q{phil@greenviewdata.com}
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.rdoc"
@@ -43,11 +43,11 @@ Gem::Specification.new do |s|
     "spec/mongar_spec.rb",
     "spec/spec_helper.rb"
   ]
-  s.homepage = "http://github.com/gdi/mongar"
+  s.homepage = %q{http://github.com/gdi/mongar}
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.10"
-  s.summary = "Replicates data from ActiveRecord (or other Ruby data mapping class) to MongoDB"
+  s.rubygems_version = %q{1.6.2}
+  s.summary = %q{Replicates data from ActiveRecord (or other Ruby data mapping class) to MongoDB}
 
   if s.respond_to? :specification_version then
     s.specification_version = 3

data/spec/fixtures/full_configure.rb

@@ -1,60 +1,96 @@
-Mongar.configure do
-  mongo :default do
-    database 'mydb'
-    user 'mongouser'
-    password 'password'
-    host '127.0.0.1'
-    port 27017
-  end
-  
-  mongo :otherdb do
-    database 'mydb'
-    user 'mongouser'
-    password 'password'
-    
-    status_collection :statuses
-  end
-  
-  replicate Domain => 'domains' do
-    use_mongodb :otherdb
-    
-    full_refresh :every => 60.minutes
-    
-    column :uri do
-      primary_index
-      transform :downcase
-    end
-    
-    column :allow_anyone_to_anyone_policy
-  end
-  
-  replicate Client do
-    column :id do
-      primary_index
-    end
-    
-    column :name
-  end
-  
-  replicate EmailAddress do
-    no_deleted_finder
-    set_updated_finder do |last_replicated_date|
-      find(:all, :conditions => ['something > ?', last_replicated_date])
-    end
-    set_created_finder do |last_replicated_date|
-      created_scope(last_replicated_date)
-    end
-    
-    full_refresh :if => Proc.new do |last_replicated_date|
-      # class eval'ed code
-      any_changes_since?(last_replicated_date)
-    end
-    
-    column :address do
-      index
-      transform do |value|
-        # some code to perform on the value
-      end
-    end
-  end
+Mongar.configure do
+  # Currently we only log to STDOUT, we have plans to use a
+  # plans to use a configurable logger.
+  # Valid log levels are :info and :debug
+  log_level :debug
+  
+  mongo :default do
+    database 'mydb'
+  end
+
+  mongo :otherdb do
+    database 'myotherdb'
+    user 'mongouser'
+    password 'password'
+    host '127.0.0.1'
+    port 27017
+    
+    # By default Mongar uses the 'statuses' collection to
+    # keep track of the last replicated times, you can specify
+    # a custom collection name
+    status_collection :stati
+  end
+  
+  # Minimal replica config to replicate the Domain model
+  # to the 'domains' mongo collection on the mongo db defined
+  # in the :default config above
+
+  replicate Domain do
+    column :name do
+      primary_index
+    end
+    
+    column :client_id
+  end
+
+  # Customized replica config to replicate the Client
+  # model to the 'customers' mongo collection on the db
+  # defined in the :otherdb config above
+  
+  replicate Client => 'customers' do
+    # Use the other mongo database 
+    use_mongodb :otherdb
+    
+    # By default, Mongar will try to determine the time on the
+    # backend database. The supported ActiveRecord database adapters
+    # are MysqlAdapter, Mysql2Adapter, and SQLServerAdapter.
+    # You can use a custom function to determine the time on Client's
+    # backend database
+    db_time_selector do
+      # this will run Client#get_db_time
+      get_db_time
+    end
+    
+    # Items are never deleted
+    no_deleted_finder
+    
+    # Customize your updated record finder.  The block is 
+    # run in the source class context
+    set_updated_finder do |last_replicated_date|
+      find(:all, :conditions => ['something > ?', last_replicated_date])
+    end
+    
+    # Custom created item finder
+    set_created_finder do |last_replicated_date|
+      created_scope(last_replicated_date)
+    end
+  
+    # Run a full refresh of all items in the database based
+    # a condition returned by the Proc
+    full_refresh :if => Proc.new do |last_replicated_date|
+      # class eval'ed code
+      any_changes_since?(last_replicated_date)
+    end
+    
+    # You can also refresh every N seconds
+    # full_refresh :every => 3600
+    
+    # Define your columns
+    column :id do
+      # The 'primary index' column is used to find items in the mongo collection
+      primary_index
+    end
+    
+    column :tag do
+      # Run the procedure :downcase on the value of Client#tag
+      transform :downcase
+    end
+    
+    column :employee_count do
+      # Run a block transformation on the value of Client#employee_count
+      transform do |value|
+        value.nil? ? 0 : value
+      end
+    end
+  end
 end

data/spec/mongar/replica_spec.rb

@@ -96,7 +96,7 @@ describe "Mongar::Replica" do
       end
       
       it "should set the last replicated at time to the time the run started" do
-        @collection.should_receive(:last_replicated_at=).with(@time)
+        @collection.should_receive(:last_replicated_at=).with(@time - 1)
         @replica.run
       end
     end

metadata

@@ -1,13 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: mongar
 version: !ruby/object:Gem::Version 
-  hash: 19
   prerelease: 
-  segments: 
-  - 0
-  - 0
-  - 6
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors: 
 - Philippe Green
@@ -15,114 +10,86 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-12-15 00:00:00 Z
+date: 2011-12-16 00:00:00 -05:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
+  name: linguistics
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
-  version_requirements: *id001
-  name: linguistics
-  prerelease: false
   type: :runtime
+  prerelease: false
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
+  name: mongo
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
-  version_requirements: *id002
-  name: mongo
-  prerelease: false
   type: :runtime
+  prerelease: false
+  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
+  name: rspec
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 2
-        - 3
-        - 0
         version: 2.3.0
-  version_requirements: *id003
-  name: rspec
-  prerelease: false
   type: :development
+  prerelease: false
+  version_requirements: *id003
 - !ruby/object:Gem::Dependency 
+  name: yard
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 0
-        - 6
-        - 0
         version: 0.6.0
-  version_requirements: *id004
-  name: yard
-  prerelease: false
   type: :development
+  prerelease: false
+  version_requirements: *id004
 - !ruby/object:Gem::Dependency 
+  name: bundler
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 1
-        - 0
-        - 0
         version: 1.0.0
-  version_requirements: *id005
-  name: bundler
-  prerelease: false
   type: :development
+  prerelease: false
+  version_requirements: *id005
 - !ruby/object:Gem::Dependency 
+  name: jeweler
   requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 1
-        - 6
-        - 4
         version: 1.6.4
-  version_requirements: *id006
-  name: jeweler
-  prerelease: false
   type: :development
+  prerelease: false
+  version_requirements: *id006
 - !ruby/object:Gem::Dependency 
+  name: rcov
   requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
-  version_requirements: *id007
-  name: rcov
-  prerelease: false
   type: :development
+  prerelease: false
+  version_requirements: *id007
 description: Replicates data from ActiveRecord (or other Ruby data mapping class) to MongoDB
 email: phil@greenviewdata.com
 executables: []
@@ -158,6 +125,7 @@ files:
 - spec/mongar/replica_spec.rb
 - spec/mongar_spec.rb
 - spec/spec_helper.rb
+has_rdoc: true
 homepage: http://github.com/gdi/mongar
 licenses: 
 - MIT
@@ -171,7 +139,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
+      hash: -627046195
       segments: 
       - 0
       version: "0"
@@ -180,14 +148,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Replicates data from ActiveRecord (or other Ruby data mapping class) to MongoDB