capistrano-mongo-sync 0.1.4 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 933c334709a5c306b67c966dc4568e9b19d08985
-  data.tar.gz: 587bbed62e4b4569e138c297b3b7425fe67e8d7a
+SHA256:
+  metadata.gz: 0f8dd6e66c59528341b95db1601e6d5ed1e9003628ab675d0ed8a19f39b00b20
+  data.tar.gz: bced25f1f251ab2aa23897cc0cb1571e61b1897f105e67565bbdd95b68ca1304
 SHA512:
-  metadata.gz: c48627b33a655b8921edc1874e2c9f48e92ce31cf5693b11ba5f1ca9552c5e3ca35c5c1d02566aef6669c8b870c8647acdaf3925da451e48eaac421d46a47a2a
-  data.tar.gz: 720a51a7c10253eebaf6c000c61d9d043f186c5961c768929ceb5381c5b8e478d3477385a091d3f0495271f661a4bfd35a99dda5abf5f632a98937cc44c79ac9
+  metadata.gz: d6d7c9bca76a43f706947f585d407210080f68dff2d4c3dc7122832b7bd51c382cd6cda73768f97f3d2cca30b8481dbc3301805a7eedc3968428bbc23f7970a2
+  data.tar.gz: a45416c9fd270069861d19c339b0ec5cd8815d0822da5aca48a32d60e8c114f3ff1925a95e1b83c37cdf5126d5cda4b4dc359110f66f64bb5256cce32a0e2264

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.1.6
+
+Add IDS feature used in conjunction with COLLECTION feature
+
 # 0.1.0
 
 Initial release

data/Gemfile.lock

@@ -1,27 +1,32 @@
 PATH
   remote: .
   specs:
-    capistrano-mongo-sync (0.1.0)
+    capistrano-mongo-sync (0.1.6)
       capistrano (~> 3.1)
       sshkit (~> 1.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    capistrano (3.4.0)
+    airbrussh (1.3.0)
+      sshkit (>= 1.6.1, != 1.7.0)
+    capistrano (3.10.1)
+      airbrussh (>= 1.0.0)
       i18n
       rake (>= 10.0.0)
-      sshkit (~> 1.3)
-    i18n (0.7.0)
+      sshkit (>= 1.9.0)
+    concurrent-ruby (1.0.5)
+    i18n (0.9.1)
+      concurrent-ruby (~> 1.0)
     metaclass (0.0.4)
     minitest (5.8.4)
     mocha (1.1.0)
       metaclass (~> 0.0.1)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
-    net-ssh (3.0.2)
-    rake (11.1.1)
-    sshkit (1.9.0)
+    net-ssh (4.2.0)
+    rake (12.3.0)
+    sshkit (1.15.1)
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
 
@@ -34,4 +39,4 @@ DEPENDENCIES
   mocha (~> 1.1)
 
 BUNDLED WITH
-   1.11.2
+   1.16.1

data/README.md

@@ -13,6 +13,12 @@ Or sync just one collection from the database:
 COLLECTION=users cap production mongo:pull
 ```
 
+Or sync some records (comma separated list of mongo id's) from one collection from the database:
+
+```ruby
+COLLECTION=users IDS=1,2,3,4 cap production mongo:pull
+```
+
 Or sync your staging database from your production database.
 
 ```ruby

data/RELEASE.md

@@ -0,0 +1,6 @@
+# Release a new version of this gem
+
+http://guides.rubygems.org/make-your-own-gem/
+
+`gem build capistrano-mongo-sync.gemspec`
+`gem push capistrano-mongo-sync-<VERSION>.gem`

data/capistrano-mongo-sync.gemspec

@@ -1,11 +1,11 @@
 Gem::Specification.new do |gem|
   gem.name        = 'capistrano-mongo-sync'
-  gem.version     = '0.1.4'
-  gem.date        = '2016-06-13'
+  gem.version     = '0.1.6'
+  gem.date        = '2018-01-18'
   gem.summary     = "A tool for keeping local mongo in sync with remote"
   gem.description = "A tool for keeping local mongo in sync with remote"
-  gem.authors     = ["Open Listings Engineering"]
-  gem.email       = 'engineering@openlistinggem.com'
+  gem.authors     = ["Open Listings Engineering", "We're hiring!"]
+  gem.email       = 'engineering@openlistings.com'
   gem.homepage    = 'https://github.com/openlistings/capistrano-mongo-sync'
   gem.license     = 'MIT'
 

data/lib/capistrano/mongo_sync/mongo_sync.rb

@@ -1,3 +1,5 @@
+require 'digest'
+
 class MongoSync
   def initialize(connection)
     @connection = connection
@@ -8,6 +10,7 @@ class MongoSync
     @staging_db = fetch(:staging_db)
     @from_db = fetch(:from_db)
     @collection = fetch(:collection) || 'full'
+    @collection_ids = fetch(:collection_ids)
     @hipchat_client = fetch(:hipchat_client)
 
     fail "Incomplete configuration: missing remote_dump_base" unless @remote_dump_base
@@ -15,11 +18,18 @@ class MongoSync
     fail "Incomplete configuration: missing production_db" unless @production_db
     fail "Incomplete configuration: missing development_db" unless @development_db
     fail "Incomplete configuration: missing from_db" unless @from_db
+    fail "Incomplete configuration: missing collection" if @collection_ids && @collection.nil?
   end
 
   # the first part of the dump dir, without the timestamp... for example "mydatabase-full"
   def dump_dir_part
-    [@from_db, @collection].join('-')
+    str = [@from_db, @collection].join('-')
+
+    if @collection_ids
+      str = [str, Digest::MD5.hexdigest(@collection_ids)].join('-')
+    end
+
+    str
   end
 
   ## Remote
@@ -40,12 +50,17 @@ class MongoSync
 
     args = ['-d', @from_db, '-o', File.join(@remote_dump_base, dump_dir)]
     args += ['-c', @collection] unless 'full' == @collection
+    args += ['-q', collection_ids_arg] if @collection_ids
 
     @connection.execute :mongodump, *args
 
     dump_dir
   end
 
+  def collection_ids_arg
+    '{_id: {$in: [%s]}}' % @collection_ids.split(',').map{|id| 'ObjectId("%s")' % id}.join(',')
+  end
+
   def staging_mongorestore!( remote_dump_dir )
     full_path_to_remote_dump_dir = if remote_dump_dir == File.basename(remote_dump_dir)
       File.join(@remote_dump_base, remote_dump_dir, @production_db)
@@ -53,7 +68,10 @@ class MongoSync
       remote_dump_dir
     end
 
-    @connection.execute :mongorestore, '--drop', '-d', @staging_db, full_path_to_remote_dump_dir
+    args = []
+    args << '--drop' if drop_collection?
+    args += ['-d', @staging_db, full_path_to_remote_dump_dir]
+    @connection.execute :mongorestore, *args
   end
 
   def last_remote_dump
@@ -144,10 +162,18 @@ class MongoSync
   def local_mongorestore!(local_dump_dir)
     db_dump_path = File.join local_dump_dir, @from_db
     @connection.within( @local_dump_base ) do
-      @connection.execute :mongorestore, '--drop', '-d', @development_db, db_dump_path
+      args = []
+      args << '--drop' if drop_collection?
+      args += ['-d', @development_db, db_dump_path]
+      @connection.execute :mongorestore, *args
     end
   end
 
+  # don't drop the collection if it's importing partially
+  def drop_collection?
+    @collection_ids.nil?
+  end
+
   ## Hipchat
   def hipchat_notify!( room, user, msg, opts = {} )
     return unless @hipchat_client

data/lib/capistrano/tasks/mongo-sync.cap

@@ -2,12 +2,14 @@
 # Usage:
 # ./bin/cap production mongo:pull
 # COLLECTION=agents ./bin/cap production mongo:pull
+# COLLECTION=agents IDS=1,2,3 ./bin/cap production mongo:pull
 # ./bin/cap production mongo:sync_prod_to_staging
 
 namespace :mongo do
   set :remote_dump_base, '/tmp/dumps'
   set :local_dump_base, '/tmp/dumps'
   set :collection, ENV['COLLECTION'] || 'full'
+  set :collection_ids, ENV['IDS']
 
   task :sync_prod_to_staging do
     set :from_db, fetch(:production_db)

data/test/test_mongo_sync.rb

@@ -13,6 +13,7 @@ class MongoSync
     @from_db = 'mydb'.freeze
     @hipchat_client = nil
     @collection = 'full'
+    @collection_ids = nil
 
     instance_variable_get("@#{x}")
   end
@@ -62,6 +63,16 @@ class MongoSyncTest < Minitest::Test
     assert_equal 'mydb-agents-2015-01-01-01-01', output_dir
   end
 
+  def test_remote_mongodump_agents_collection_with_collection_ids
+    @mongo_sync.instance_variable_set("@collection", 'agents')
+    @mongo_sync.instance_variable_set("@collection_ids", '1,2,3')
+    @connection.expects(:execute).once().with(:mongodump, '-d', 'mydb', '-o', '/mnt/tmp/dumps/mydb-agents-55b84a9d317184fe61224bfb4a060fb0-2015-01-01-01-01', '-c', 'agents', '-q', '{_id: {$in: [ObjectId("1"),ObjectId("2"),ObjectId("3")]}}')
+    output_dir = @mongo_sync.remote_mongodump!
+
+    # path includes hexdigest of IDS string
+    assert_equal 'mydb-agents-55b84a9d317184fe61224bfb4a060fb0-2015-01-01-01-01', output_dir
+  end
+
   def test_dump_prompt_message
     path_to_tgz = '/tmp/dumps/mydb-full-2015-01-01-01-00.tgz'
     expected_msg = "Use local dump from today at 01:00 AM? \"%s\"? (y/n)" % path_to_tgz
@@ -104,7 +115,7 @@ class MongoSyncTest < Minitest::Test
   def test_last_remote_dump_no_dumps
     @connection.expects(:test, 'ls -td /mnt/tmp/dumps/*/mydb').returns(false)
     lrd = @mongo_sync.last_remote_dump
-    assert_equal nil, lrd
+    assert_nil lrd
   end
 
   def test_last_remote_dump_dumps_y
@@ -126,19 +137,19 @@ class MongoSyncTest < Minitest::Test
     @mongo_sync.instance_variable_set '@use_remote_dump_dir', 'n'
 
     lrd = @mongo_sync.last_remote_dump
-    assert_equal nil, lrd
+    assert_nil lrd
   end
 
   def test_last_local_dump_no_dumps
     @connection.expects(:test, 'ls -td /mnt/tmp/dumps/mydb-full*/mydb').returns(false)
     lrd = @mongo_sync.last_remote_dump
-    assert_equal nil, lrd
+    assert_nil lrd
   end
 
   def test_last_remote_dump_tgz
     @connection.expects(:test, 'ls -td /mnt/tmp/dumps/mydb-full*/mydb').returns(false)
     lrd = @mongo_sync.last_remote_dump
-    assert_equal nil, lrd
+    assert_nil lrd
   end
 
   def test_local_mongorestore

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: capistrano-mongo-sync
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.6
 platform: ruby
 authors:
 - Open Listings Engineering
+- We're hiring!
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-13 00:00:00.000000000 Z
+date: 2018-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capistrano
@@ -67,7 +68,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.1'
 description: A tool for keeping local mongo in sync with remote
-email: engineering@openlistinggem.com
+email: engineering@openlistings.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -78,10 +79,10 @@ files:
 - Gemfile.lock
 - LICENSE
 - README.md
+- RELEASE.md
 - Rakefile
 - capistrano-mongo-sync.gemspec
 - lib/capistrano/mongo-sync.rb
-- lib/capistrano/mongo-sync/mongo-sync.cap
 - lib/capistrano/mongo_sync/mongo_sync.rb
 - lib/capistrano/tasks/mongo-sync.cap
 - test/test_mongo_sync.rb
@@ -105,7 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: A tool for keeping local mongo in sync with remote

data/lib/capistrano/mongo-sync/mongo-sync.cap

@@ -1,75 +0,0 @@
-##
-# Usage:
-# ./bin/cap production mongo:pull
-# COLLECTION=agents ./bin/cap production mongo:pull
-# ./bin/cap production mongo:sync_prod_to_staging
-
-namespace :mongo do
-  set :remote_dump_base, '/tmp/dumps'
-  set :local_dump_base, '/tmp/dumps'
-  set :collection, ENV['COLLECTION'] || 'full'
-
-  task :sync_prod_to_staging do
-    set :from_db, fetch(:production_db)
-
-    on roles(:db) do
-      ms_remote = MongoSync.new(self)
-      ms_remote.hipchat_notify! 'Engineering', 'capistrano', 'Started syncing prod to staging...', color: 'yellow'
-      ms_remote.remote_setup!
-      ms_remote.remote_cleanup!
-
-      remote_dump_dir = ms_remote.last_remote_dump
-      remote_dump_dir ||= ms_remote.remote_mongodump!
-
-      ms_remote.staging_mongorestore! remote_dump_dir
-      ms_remote.hipchat_notify! 'Engineering', 'capistrano', 'Finished syncing prod to staging...', color: 'green'
-    end
-  end
-
-  task :pull do
-    set :from_db, :production == fetch(:stage) ? fetch(:production_db) : fetch(:staging_db)
-
-    run_locally do
-      ms_local = MongoSync.new(self)
-      ms_local.local_setup!
-      ms_local.local_cleanup!
-
-      # variable scope
-      remote_tgz = nil
-      local_tgz = nil
-
-      if local_tgz = ms_local.last_local_dump
-        # use local tgz
-        ms_local.local_unarchive! local_tgz
-
-        local_dump_dir = File.join fetch(:local_dump_base), File.basename(local_tgz, '.tgz')
-        ms_local.local_mongorestore!(local_dump_dir)
-      else
-        # get dump from remote
-        on roles(:db) do
-          ms_remote = MongoSync.new(self)
-          ms_remote.remote_setup!
-          ms_remote.remote_cleanup!
-
-          # find & choose tarfile or dump & archive
-          remote_tgz = ms_remote.last_remote_dump_tgz
-          unless remote_tgz
-            dump_dir = ms_remote.remote_mongodump!
-            remote_tgz = ms_remote.remote_archive! dump_dir
-          end
-
-          # download!
-          local_tgz = File.join fetch(:local_dump_base), File.basename(remote_tgz)
-          download! remote_tgz, local_tgz, method: :scp
-        end
-
-        # unarchive!
-        ms_local.local_unarchive!(local_tgz)
-
-        # restore!
-        local_dump_dir = File.join fetch(:local_dump_base), File.basename(remote_tgz, '.tgz')
-        ms_local.local_mongorestore!(local_dump_dir)
-      end
-    end
-  end
-end