dbsync 0.3.0 → 1.0.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4d8a3ddf36966a8c9aec5c43f62b383f6a01dac4
-  data.tar.gz: 04dea7b42bca9c6492be1153947ca10975bca739
+  metadata.gz: 6361e83ca4c1d35147ad652e0be945ee8d198cce
+  data.tar.gz: a0c7d17fc882430f53c2309f14a9be2f379728c0
 SHA512:
-  metadata.gz: 2a0119d2c7b4f67f5e7ec95ebcf29000a6692a7f8c0fd1523703da10efcb71b827c4d5debc347858a086e52a1261b5d5bdb372aae27aadd59a8bee9c9d9de7dc
-  data.tar.gz: 7481b11c17c29b718c6b4f7addc1ae1345e7e468a012a10b23c0ff693e8a53b815f788fef3eb83312bb93e684b5557ae8ccf86fce23215e6624ddb53f5e974d7
+  metadata.gz: 126d7451021ddf112e89a6c753c185b470187d03fe26cd965b26b046a0fc3f652e21be03eb0e379a9871de85c7ada0a0dde5b69381c8ea0e233bfc708522cac0
+  data.tar.gz: 075b0e1041f00c29938133e08b3c4d59342fa45f8970b6350f0707263e12ec5c3acc377fd5d611e60653dac825be5e567dd2ebc32ed2aa342f97d52fc24df61e

data/CHANGELOG.md

@@ -1,4 +1,11 @@
-## 0.3.0 (unreleased)
+## 1.0.0.beta2
+* Remove old dependency
+
+## 1.0.0.beta1
+* Added curl support
+* Removed `clone_dump` task (use `fetch`)
+
+## 0.3.0
 * Removed all Rails dependencies. Rails is now optional.
 * Removed scp usage. Everything uses rsync now.
 * Added Dbsync.file_config and Dbsync.db_config to manually set configuration.

data/Gemfile

@@ -1,4 +1,7 @@
 source "http://rubygems.org"
 
+gem 'pry'
+gem 'rspec'
+
 # Specify your gem's dependencies in dbsync.gemspec
 gemspec

data/README.md

@@ -24,8 +24,8 @@ Add the following to your `config/environments/development.rb` file. Depending o
 
 ```ruby
 config.dbsync = {
-  :remote => '66.123.4.567:~/dbsync/mydb.dump',
-  :local  => '../dbsync/mydb.dump'
+  :remote => 'dbuser@66.123.4.567:~/dbsync/mydb.dump',
+  :local  => '../dbsync/dbsync-yourapp.dump'
 }
 ```
 
@@ -37,12 +37,12 @@ You can also specify the dbsync configuration with `Dbsync.file_config` and `Dbs
 
 ```ruby
 Dbsync.file_config = {
-  :local => "../dbsync/dbsync.dump",
+  :local => "../dbsync/dbsync-yourapp.dump",
   :remote => "dbuser@100.0.100.100:~dbuser/dbsync.dump"
 }
 
 Dbsync.db_config = {
-  :adapter  => "mysql2", # Not actually used yet
+  :adapter  => "mysql2",
   :database => "yourdb",
   :username => "youruser",
   :password => "yourcoolpassword"
@@ -66,7 +66,6 @@ Run `rake -T dbsync` for all of the available tasks. The tasks are named after `
 ```
 rake dbsync             # Alias for dbsync:pull
 rake dbsync:clone       # Copy the remote dump file, reset the local database, and load in the dump file
-rake dbsync:clone_dump  # Copy the remote dump file to a local destination
 rake dbsync:config      # Show the dbsync configuration
 rake dbsync:fetch       # Update the local dump file from the remote source
 rake dbsync:merge       # Update the local database with the local dump file
@@ -75,8 +74,36 @@ rake dbsync:reset       # Drop and Create the database, then load the dump file
 ```
 
 
-### Caveats
+### Download strategies
+`curl` and `rsync` are currently the only two supported options. You can pass a `strategy` option to the dbsync config (`:curl` or `:rsync`) to explicitly specify which strategy to use, or Dbsync will try to infer the strategy. Right now the strategy inference is a little dodgy, so it's best to just specify explicitly.
+
+```ruby
+config.dbsync = {
+  :strategy   => :rsync,
+  :remote     => 'username@66.123.4.567:~/dbsync/mydb.dump',
+  :local      => '../dbsync/mydb.dump'
+}
+
+config.dbsync = {
+  :strategy   => :curl,
+  :bin_opts   => "--netrc",
+  :remote     => 'ftp://ftp.yourserver.com/dbsync/mydb.dump',
+  :local      => '../dbsync/mydb.dump'
+}
+```
+
+`bin_opts` will be passed directly to the bin command.
+
+
+### Compressed files
+Dbsync will attempt to determine if it needs to uncompress your file. `tar` and `gz` files are currently supported.
 
+
+### Database
+Currently only MySQL is supported.
+
+
+### Caveats
 * The `merge` process doesn't clear out your database first. This is to improve performance. Therefore, any tables which you removed on the remote host won't be removed locally. To do a complete reset of your database, run `rake dbsync:reset`. This resets your database (`db:drop` and `db:create`), and then merges in the local file.
 * Rails: the test database isn't automatically updated when syncing to your development database. After a `dbsync` and before you run tests, you'll need to run `rake db:test:prepare` to setup your database.
 * Rails: your schema.rb isn't involed in `dbsync` at all. You need to manage it yourself.

data/dbsync.gemspec

@@ -19,5 +19,5 @@ Gem::Specification.new do |s|
 
   s.add_dependency "cocaine", [">= 0.5.0", "< 0.6"]
 
-  s.add_development_dependency "rspec", '~> 0'
+  s.add_development_dependency "bundler", ['>= 1.6', "< 2"]
 end

data/lib/dbsync.rb

@@ -1,7 +1,16 @@
+require 'cocaine'
+require 'fileutils'
+
 require "dbsync/version"
+require 'dbsync/util'
+require "dbsync/strategy"
+require "dbsync/importer"
 require 'dbsync/sync'
 
 module Dbsync
+  class ConfigError < StandardError
+  end
+
   if defined?(Rails)
     class Railtie < Rails::Railtie
       rake_tasks do

data/lib/dbsync/importer.rb

@@ -0,0 +1,2 @@
+require 'dbsync/importer/base'
+require 'dbsync/importer/mysql'

data/lib/dbsync/importer/base.rb

@@ -0,0 +1,13 @@
+module Dbsync
+  module Importer
+    class Base
+      def initialize(db_config, local)
+        @db_config  = db_config
+        @local      = local
+      end
+
+      def merge
+      end
+    end
+  end
+end

data/lib/dbsync/importer/mysql.rb

@@ -0,0 +1,26 @@
+module Dbsync
+  module Importer
+    class Mysql < Base
+      def merge
+        username  = @db_config[:username]
+        password  = @db_config[:password]
+        host      = @db_config[:host]
+        database  = @db_config[:database]
+
+        opts = ""
+        opts += "-u :username " if username
+        opts += "-p:password "  if password
+        opts += "-h :host "     if host
+
+        line = Cocaine::CommandLine.new('mysql', "#{opts} :database < :local")
+        line.run({
+          :username   => username,
+          :password   => password,
+          :host       => host,
+          :database   => database,
+          :local      => @local
+        })
+      end
+    end
+  end
+end

data/lib/dbsync/strategy.rb

@@ -0,0 +1,3 @@
+require 'dbsync/strategy/base'
+require 'dbsync/strategy/rsync'
+require 'dbsync/strategy/curl'

data/lib/dbsync/strategy/base.rb

@@ -0,0 +1,20 @@
+require 'cocaine'
+require 'fileutils'
+
+module Dbsync
+  module Strategy
+    class Base
+      def initialize(remote, local, bin_opts)
+        @remote     = remote
+        @local      = local
+        @bin_opts   = bin_opts
+      end
+
+
+      # Strategy interface
+      # Retrieve the dump file.
+      def fetch
+      end
+    end
+  end
+end

data/lib/dbsync/strategy/curl.rb

@@ -0,0 +1,16 @@
+module Dbsync
+  module Strategy
+    class Curl < Base
+      BIN = "curl"
+
+      def fetch
+        line = Cocaine::CommandLine.new(BIN, ':remote :bin_opts > :local')
+        line.run({
+          :bin_opts   => @bin_opts,
+          :remote     => @remote,
+          :local      => @local
+        })
+      end
+    end
+  end
+end

data/lib/dbsync/strategy/rsync.rb

@@ -0,0 +1,16 @@
+module Dbsync
+  module Strategy
+    class Rsync < Base
+      BIN = "rsync"
+
+      def fetch
+        line = Cocaine::CommandLine.new(BIN, ':bin_opts :remote :local')
+        line.run({
+          :bin_opts   => @bin_opts,
+          :remote     => @remote,
+          :local      => @local
+        })
+      end
+    end
+  end
+end

data/lib/dbsync/sync.rb

@@ -3,85 +3,48 @@ require 'fileutils'
 
 module Dbsync
   class Sync
-    class << self
-      def notify(message="")
-        $stdout.puts "[#{Time.now.strftime('%T')}] [dbsync] #{message}"
-      end
-    end
+    STRATEGY = {
+      :rsync    => Dbsync::Strategy::Rsync,
+      :curl   => Dbsync::Strategy::Curl
+    }
+
+    IMPORTER = {
+      :mysql => Dbsync::Importer::Mysql
+    }
 
 
-    def initialize(ssh_config, db_config, options={})
-      ssh_config  = symbolize_keys(ssh_config)
-      db_config   = symbolize_keys(db_config)
+    def initialize(file_config, db_config, options={})
+      @file_config  = Dbsync::Util.symbolize_keys(file_config)
+      @db_config   = Dbsync::Util.symbolize_keys(db_config)
 
       @verbose = !!options[:verbose]
 
-      @db_username  = db_config[:username]
-      @db_password  = db_config[:password]
-      @db_host      = db_config[:host]
-      @db_database  = db_config[:database]
-
-      @remote   = ssh_config[:remote]
-      @local    = File.expand_path(ssh_config[:local]) if ssh_config[:local]
-
-      if !@remote
-        $stdout.puts "DEPRECATED: The remote_host, remote_dir, and filename " \
-          "options will be removed. " \
-          "Instead, combine remote_host, remote_dir, and filename into a " \
-          "single 'remote' configuration. Example: " \
-          "'{ remote: \"dbuser@100.0.1.100:~/dbuser/yourdb.dump\" }'"
-
-        remote_host = ssh_config[:remote_host]
-        remote_dir  = ssh_config[:remote_dir]
-        filename    = ssh_config[:filename]
-        @remote = "#{remote_host}:#{File.join(remote_dir, filename)}"
-      end
+      @remote           = @file_config[:remote]
+      remote_filename   = File.basename(@remote)
 
-      if !@local
-        $stdout.puts "DEPRECATED: The local_dir and filename " \
-          "options will be removed. " \
-          "Instead, combine local_dir and filename into a " \
-          "single 'local' configuration. Example: " \
-          "'{ local: \"../dbsync/yourdb.dump\" }'"
+      @local          = File.expand_path(@file_config[:local])
+      local_dir       = File.dirname(@local)
+      @download       = File.join(local_dir, remote_filename)
 
-        local_dir = ssh_config[:local_dir]
-        filename  = ssh_config[:filename]
-        @local  = File.expand_path(File.join(local_dir, filename))
-      end
+      FileUtils.mkdir_p(local_dir)
+
+      @strategy = strategy.new(@remote, @download, @file_config[:bin_opts])
+      @importer = importer.new(@db_config, @local)
     end
 
 
     # Update the local dump file from the remote source (using rsync).
     def fetch
-      notify "Updating '#{@local}' from '#{@remote}' via rsync..."
-
-      FileUtils.mkdir_p(File.dirname(@local))
-
-      line = Cocaine::CommandLine.new('rsync', '-v :remote :local')
-      line.run({
-        :remote => @remote,
-        :local  => @local
-      })
+      notify "Downloading..."
+      @strategy.fetch
+      extract
     end
 
 
     # Update the local database with the local dump file.
     def merge
-      notify "Dumping data from '#{@local}' into '#{@db_database}'..."
-
-      options = ""
-      options += "-u :username " if @db_username
-      options += "-p:password "  if @db_password
-      options += "-h :host "     if @db_host
-
-      line = Cocaine::CommandLine.new('mysql', "#{options} :database < :local")
-      line.run({
-        :username   => @db_username,
-        :password   => @db_password,
-        :host       => @db_host,
-        :database   => @db_database,
-        :local      => @local
-      })
+      notify  "Importing..."
+      @importer.merge
     end
 
 
@@ -92,36 +55,61 @@ module Dbsync
     end
 
 
-    # Copy the remote dump file to a local destination.
-    # Instead of requiring two different tools (rsync and scp) for this
-    # library, instead we'll just remove the local file if it exists
-    # then run rsync, which is essentially a full copy.
-    def clone_dump
-      notify "Copying '#{@remote}' into '#{@local}' via rsync..."
-      FileUtils.rm_f(@local)
-      fetch
+    private
+
+    # TODO: There is a ruby library called "Archive" which can
+    # extract these much better for us. The only problem is that
+    # we can't specify a *filename* to extract to, which is
+    # important for us.
+    def extract
+      case @download
+      when /\.tar/   then untar
+      when /\.gz\z/  then gunzip
+      end
     end
 
+    # We're overwriting files by default. Is this okay? Probably not.
+    def gunzip
+      line = Cocaine::CommandLine.new('gunzip', "-c :download > :local")
+      line.run(download: @download, local: @local)
+    end
 
-    private
+    def untar
+      line = Cocaine::CommandLine.new('tar', "-C :local -xf :download")
+      line.run(download: @download, local: @local)
+    end
 
-    def symbolize_keys(hash)
-      return hash unless hash.keys.any? { |k| k.is_a?(String) }
 
-      result = {}
-      hash.each_key { |k| result[k.to_sym] = hash[k] }
-      result
+    def importer
+      IMPORTER[@file_config[:importer]] ||
+      IMPORTER[infer_importer_key]
     end
 
+    def infer_importer_key
+      case @db_config[:adapter]
+      when /mysql/ then :mysql
+      else raise Dbsync::ConfigError, "Only MySQL supported right now."
+      end
+    end
 
-    def raise_missing(config="")
-      raise "Missing Configuration: '#{config}'. " \
-            "See README for required config."
+
+    def strategy
+      STRATEGY[@file_config[:strategy]] ||
+      STRATEGY[infer_strategy_key]
     end
 
+    # These matches could be a lot better.
+    def infer_strategy_key
+      case @remote
+      when /\A.+?@.+?:.+?/ then :rsync
+      else :curl
+      end
+    end
+
+
     def notify(*args)
       if @verbose
-        Sync.notify(*args)
+        Dbsync::Util.notify(*args)
       end
     end
   end

data/lib/dbsync/util.rb

@@ -0,0 +1,17 @@
+module Dbsync
+  module Util
+    class << self
+      def symbolize_keys(hash)
+        return hash unless hash.keys.any? { |k| k.is_a?(String) }
+
+        result = {}
+        hash.each_key { |k| result[k.to_sym] = hash[k] }
+        result
+      end
+
+      def notify(message="")
+        $stdout.puts "[#{Time.now.strftime('%T')}] [dbsync] #{message}"
+      end
+    end
+  end
+end

data/lib/dbsync/version.rb

@@ -1,3 +1,3 @@
 module Dbsync
-  VERSION = "0.3.0"
+  VERSION = "1.0.0.beta2"
 end

data/lib/tasks/dbsync.rake

@@ -13,7 +13,7 @@ end
 namespace :dbsync do
   task :setup => :environment do
     if defined?(Rails)
-      Dbsync::Sync.notify "Rails Environment: #{Rails.env}"
+      Dbsync::Util.notify "Rails Environment: #{Rails.env}"
 
       if Rails.env == 'production'
         raise "These tasks are destructive and shouldn't " \
@@ -29,7 +29,7 @@ namespace :dbsync do
   task :config => :setup do
     # We don't use Sync.notify here because we don't want or need
     # the extra output that comes with it.
-    $stdout.puts file_config.to_yaml
+    Dbsync::Util.notify file_config
   end
 
 
@@ -42,7 +42,7 @@ namespace :dbsync do
   desc  "Copy the remote dump file, reset the local database, " \
         "and load in the dump file"
   task :clone => :setup do
-    @dbsync.clone_dump
+    @dbsync.fetch
     Rake::Task['dbsync:reset'].invoke
   end
 
@@ -53,12 +53,6 @@ namespace :dbsync do
   end
 
 
-  desc "Copy the remote dump file to a local destination"
-  task :clone_dump => :setup do
-    @dbsync.clone_dump
-  end
-
-
   desc "Update the local database with the local dump file."
   task :merge => :setup do
     @dbsync.merge
@@ -80,11 +74,11 @@ end
 def db_config
   return Dbsync.db_config if Dbsync.db_config
   return ActiveRecord::Base.configurations[Rails.env] if defined?(Rails)
-  raise "No database configuration found."
+  raise Dbsync::ConfigError, "No database configuration found."
 end
 
 def file_config
   return Dbsync.file_config if Dbsync.file_config
   return Rails.application.config.dbsync if defined?(Rails)
-  raise "No remote configuration found."
+  raise Dbsync::ConfigError, "No remote configuration found."
 end

data/spec/sync_spec.rb

@@ -46,19 +46,6 @@ describe Dbsync::Sync do
   describe '#merge' do
   end
 
-  describe '#clone_dump' do
-    it "removes the local file and rsyncs a fresh one" do
-      # Put a dummy local file in place to make sure it gets removed
-      File.open(local_path, "w") { |f| f.write "Hello." }
-      File.read(local_path).should eq "Hello."
-
-      sync = Dbsync::Sync.new(ssh_config, db_config)
-      sync.clone_dump
-
-      File.read(local_path).should eq File.read(remote_path)
-    end
-  end
-
   describe '#pull' do
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dbsync
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.0.0.beta2
 platform: ruby
 authors:
 - Bryan Ricker
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-15 00:00:00.000000000 Z
+date: 2014-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cocaine
@@ -31,19 +31,25 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.6'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 description: A set of rake tasks to help you sync your remote production data with
   your local database for development.
 email:
@@ -60,7 +66,15 @@ files:
 - Rakefile
 - dbsync.gemspec
 - lib/dbsync.rb
+- lib/dbsync/importer.rb
+- lib/dbsync/importer/base.rb
+- lib/dbsync/importer/mysql.rb
+- lib/dbsync/strategy.rb
+- lib/dbsync/strategy/base.rb
+- lib/dbsync/strategy/curl.rb
+- lib/dbsync/strategy/rsync.rb
 - lib/dbsync/sync.rb
+- lib/dbsync/util.rb
 - lib/dbsync/version.rb
 - lib/tasks/dbsync.rake
 - spec/local/.gitkeep
@@ -82,9 +96,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.2.2