db_replicator 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ef06bcc9e687a964998b09e63bd2bb71aeba5d6d
+  data.tar.gz: 1650f534c1e128d84d1d14b5da94a40f9581e18c
+SHA512:
+  metadata.gz: 62a47ea6112545a0a5e472f103fb127d3b5b71a24c312224cdfe91c77e780dafa041cc7ca69b0fd8d95aaa12d6de501760ebe6c5084bf352351f998eb26dce15
+  data.tar.gz: ec28276dfcfaede3eb2ccbce292d61243e3fc111153231521f0f7059abcc4da99198858f6a14332b1009123e74be9bdc57d83049cc804d89c53460061aa37454

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in db_replicator.gemspec
+gemspec
+gem 'ruby-progressbar'
+gem 'net-ssh'
+gem 'net-scp'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Michael Eatherly
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+# DbReplicator
+
+This gem allow you to down load and import your production db to test with. 
+
+As of now I only suport MySQL. I plan to add more adapters in the future. See TODO below.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'db_replicator'
+```
+
+And then execute:
+
+    $ rails g db_replicator:install
+
+## Usage
+
+You can download your production db by running:
+
+    $ rake dbr:prod_to_local
+
+
+## TODO
+
+* Allow secure upload of production db to another enviroment. e.g. staging
+* Add more adapters. 
+
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/db_replicator/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new('spec')
+
+task :default => :spec
+

data/db_replicator.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'db_replicator/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "db_replicator"
+  spec.version       = DbReplicator::VERSION
+  spec.authors       = ["Michael Eatherly"]
+  spec.email         = ["meatherly@gmail.com"]
+  spec.summary       = ""
+  spec.description   = ""
+  spec.homepage      = "https://github.com/meatherly/db_replicator"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "generator_spec"
+  spec.add_development_dependency "rspec"
+  spec.add_runtime_dependency 'sequel', '~> 4.15.0'
+  spec.add_runtime_dependency 'ruby-progressbar'
+  spec.add_runtime_dependency 'rails', '>= 4'
+  spec.add_runtime_dependency 'colorize'
+
+  spec.required_ruby_version = '~> 2.0'
+
+end

data/lib/db_replicator/configuration.rb

@@ -0,0 +1,12 @@
+require 'etc'
+
+module DbReplicator
+  class Configuration
+    attr_accessor :proxy_host, :user
+
+    def initialize
+      @proxy_host = 'example.com'
+      @user = Etc.getlogin
+    end
+  end
+end

data/lib/db_replicator/downloader.rb

@@ -0,0 +1,25 @@
+require 'net/ssh'
+require 'net/scp'
+require 'ruby-progressbar'
+module DbReplicator
+  class Downloader
+    def self.download_production
+      pp DbReplicator.prod_db_configs['database']
+      Net::SSH.start(DbReplicator.configuration.proxy_host, DbReplicator.configuration.user) do |session|
+        DbReplicator.document_action "Creating MySQL dump file.", "Create complete." do
+          session.exec! "mysqldump -u #{DbReplicator.prod_db_configs['user']} --password=#{DbReplicator.prod_db_configs['password']} #{DbReplicator.prod_db_configs['database']} > #{DbReplicator.dump_file_name}"
+        end
+        DbReplicator.document_action "Downloading MySQL dump file", "Download complete." do
+          pb = ProgressBar.create(:format => '%t %B %p%% %a')
+          session.scp.download!(DbReplicator.dump_file_name, DbReplicator.dumps_dir) do |ch, name, sent, total|
+            pb.total = total
+            pb.progress = sent
+          end
+        end
+        DbReplicator.document_action "Deleting MySQL dump file on #{DbReplicator.configuration.proxy_host}", "Delete complete." do
+          session.exec! "rm -f #{DbReplicator.dump_file_name}"
+        end
+      end
+    end
+  end
+end

data/lib/db_replicator/importer.rb

@@ -0,0 +1,62 @@
+module DbReplicator
+  class Importer
+    attr_accessor :to_db_env, :db_env
+    def initialize(to_db_env)
+      @to_db_env = to_db_env
+      @to_db_configs = DbReplicator.db_configs(@to_db_env)
+      if @to_db_configs['adapter'] == 'sqlite3'
+        @tmp_db_configs = create_tmp_db_configs
+      else
+        @tmp_db_configs = nil
+      end
+    end
+
+    def import_db!
+      DbReplicator.document_action "Executing db:drop && db:create to get a fresh database", "Fresh database created." do
+        create_fresh_db
+      end
+      DbReplicator.document_action "Importing mysql dump to #{@to_db_configs['database']} database", "Import complete." do
+        pp @tmp_db_configs
+        if @tmp_db_configs
+          DbReplicator.document_action "Transfering db to sqlite3", "Transfer complete" do
+            convert_sql_dump_and_import
+          end
+        else
+          puts "Executing: mysql -u root #{@to_db_configs['database']} < #{DbReplicator.dump_file}"
+          exec "mysql -u root #{@to_db_configs['database']} < #{DbReplicator.dump_file}"
+        end
+      end
+      DbReplicator.document_action "Executing db:migrate to update database Just in case their are pending migrations", "Migrate complete." do
+        system "bundle exec rake db:migrate"
+      end
+    end
+    
+    private
+
+    def convert_sql_dump_and_import
+        DbReplicator.document_action "Creating temp db for transfer #{create_tmp_db_configs['database']}", "Create complete" do
+          ActiveRecord::Tasks::DatabaseTasks.create(create_tmp_db_configs)
+        end        
+        DbReplicator.document_action "Importing data to temp db. DB: #{create_tmp_db_configs['database']}; File: #{DbReplicator.dump_file}", "Import complete" do
+          puts "Executing: mysql -u root #{create_tmp_db_configs['database']} < #{DbReplicator.dump_file}"
+          system "mysql -u root #{create_tmp_db_configs['database']} < #{DbReplicator.dump_file}"
+        end
+        DbReplicator.document_action "Starting data transfer", "Data transfer complete." do
+          puts "Executing: sequel #{DbReplicator.prod_db_configs['adapter']}://localhost/#{create_tmp_db_configs['database']}?user=root -C sqlite://#{@to_db_configs['database']}"
+          system "sequel #{DbReplicator.prod_db_configs['adapter']}://localhost/#{create_tmp_db_configs['database']}?user=root -C sqlite://#{@to_db_configs['database']}" 
+        end      
+    end
+
+    def create_tmp_db_configs
+      tmp_configs = @to_db_configs.clone
+      tmp_configs['database'] = "#{DbReplicator.prod_db_configs['database']}_db_replicator"
+      # tmp_configs['host'] = 'localhost'
+      tmp_configs.reject!{|k, v| ['password', 'username'].include?(k.to_s) || v.nil? }
+      tmp_configs
+    end
+
+    def create_fresh_db
+      system "bundle exec rake db:drop db:create"
+    end
+  end
+end

data/lib/db_replicator/tasks.rb

@@ -0,0 +1,6 @@
+require 'rubygems'
+require 'rake'
+# Make tasks visible for Rails also when used as gem.
+Dir[File.join(File.dirname(__FILE__), '..', 'tasks', '**/*.rake')].each { |rake| load rake }
+Dir[File.join(File.dirname(__FILE__), '..', '..', 'tasks', '**/*.rake')].each { |rake| load rake }
+

data/lib/db_replicator/version.rb

@@ -0,0 +1,3 @@
+module DbReplicator
+  VERSION = "0.0.1"
+end

data/lib/db_replicator.rb

@@ -0,0 +1,50 @@
+$:.unshift(File.dirname(__FILE__))
+require 'db_replicator/version'
+require 'db_replicator/tasks'
+require 'db_replicator/configuration'
+require 'db_replicator/downloader'
+require 'db_replicator/importer'
+require 'colorize'
+
+module DbReplicator
+  class << self
+    attr_writer :configuration
+
+    
+  end
+
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.configure
+    yield(configuration)
+  end
+
+  def self.document_action(before, after, &block)
+    puts before.colorize(:yellow)
+    yield block
+    puts after.colorize(:green)
+  end
+
+  def self.db_configs(db_env=Rails.env)
+    ActiveRecord::Base.configurations[db_env]
+  end
+
+  def self.dump_file_name
+    ActiveRecord::Base.configurations['production']['database'] + '.sql'
+  end
+
+  def self.dumps_dir
+    File.join Rails.root, '.db_replicator_dumps'
+  end
+
+  def self.dump_file
+    File.join(dumps_dir, dump_file_name)
+  end
+
+  def self.prod_db_configs
+    @prod_configs ||= DbReplicator.db_configs('production')
+  end
+
+end

data/lib/generators/db_replicator/install_generator.rb

@@ -0,0 +1,25 @@
+module DbReplicator
+  class InstallGenerator < Rails::Generators::Base
+    source_root File.expand_path("../../templates", __FILE__)
+    desc "Creates the base config file for DbReplicator, the .db_replicator_dumps directory, and adds .db_replicator_dumps to your .gitignore file."
+
+    def copy_initializer
+      puts destination_root
+      template "db_replicator.rb", "config/initializers/db_replicator.rb"
+    end
+
+    def create_dumps_dir
+      empty_directory ".db_replicator_dumps"
+    end
+
+    def add_dumps_dir_to_gitignore
+      if File.exists?(File.join(destination_root, '.gitignore'))
+        append_to_file '.gitignore' do
+          '.db_replicator_dumps'
+        end
+      else
+        create_file ".gitignore", ".db_replicator_dumps"
+      end
+    end
+  end
+end

data/lib/generators/templates/db_replicator.rb

@@ -0,0 +1,4 @@
+DbReplicator.configure do |config|
+  # The proxy_host is a server that can connect to your production db.
+  config.proxy_host = 'www.example.com'
+end

data/lib/tasks/dbr.rake

@@ -0,0 +1,21 @@
+db_replicator_lib = File.expand_path(File.dirname(File.dirname(__FILE__)))
+require "#{db_replicator_lib}/db_replicator"
+
+namespace :dbr do
+  desc "Imports the production db into the current machines environment db"
+  task prod_to_local: [:environment, :download_prod_db] do
+    importer = DbReplicator::Importer.new('development')
+    importer.import_db!
+    puts "******* You now have production data in your current database *******"
+  end
+
+  desc "Download the production db dump to ~/mysql_dumps"
+  task download_prod_db: :environment do
+    DbReplicator::Downloader.download_production
+  end
+end
+
+
+
+
+

data/spec/db_replicator/configuration_spec.rb

@@ -0,0 +1,13 @@
+require 'etc'
+require 'spec_helper'
+
+module DbReplicator
+  describe Configuration do
+    describe "#proxy_host" do
+      it 'default values' do
+        Configuration.new.proxy_host.should eq 'example.com'
+        Configuration.new.user.should eq Etc.getlogin
+      end
+    end
+  end
+end

data/spec/db_replicator_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+describe DbReplicator do
+
+  describe "#configure" do
+    before do
+      DbReplicator.configure do |config|
+        config.proxy_host = 'www.example.com'
+        config.user = 'user'
+      end
+    end
+
+    it 'returns the custom values' do
+      DbReplicator.configuration.proxy_host.should == 'www.example.com'
+      DbReplicator.configuration.user.should == 'user'
+    end
+  end
+end

data/spec/generators/db_replicator/install_generator_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+require 'generator_spec/test_case'
+require 'generator_spec'
+require 'generators/db_replicator/install_generator'
+
+describe DbReplicator::InstallGenerator, type: :generator do
+  include GeneratorSpec::TestCase
+  destination File.expand_path("../../tmp", __FILE__)
+
+  before do
+    prepare_destination
+    run_generator
+  end
+
+  it 'should have created the initializer with the correct contents' do
+    assert_file "config/initializers/db_replicator.rb"
+  end
+
+  it 'should create the SQL dumps directory' do
+    destination_root.should have_structure {
+      directory ".db_replicator_dumps"
+    }    
+  end
+
+  it 'should add .db_replicator_dumps to the .gitignore file' do
+    destination_root.should have_structure {
+      file ".gitignore" do
+        ".db_replicator_dumps"
+      end
+    }  
+  end
+
+end

data/spec/generators/tmp/.gitignore

@@ -0,0 +1 @@
+.db_replicator_dumps

data/spec/generators/tmp/config/initializers/db_replicator.rb

@@ -0,0 +1,4 @@
+DbReplicator.configure do |config|
+  # The proxy_host is a server that can connect to your production db.
+  config.proxy_host = 'www.example.com'
+end

data/spec/rails_app/README.rdoc

@@ -0,0 +1,28 @@
+== README
+
+This README would normally document whatever steps are necessary to get the
+application up and running.
+
+Things you may want to cover:
+
+* Ruby version
+
+* System dependencies
+
+* Configuration
+
+* Database creation
+
+* Database initialization
+
+* How to run the test suite
+
+* Services (job queues, cache servers, search engines, etc.)
+
+* Deployment instructions
+
+* ...
+
+
+Please feel free to use a different markup language if you do not plan to run
+<tt>rake doc:app</tt>.

data/spec/rails_app/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require File.expand_path('../config/application', __FILE__)
+
+Rails.application.load_tasks

data/spec/rails_app/app/assets/javascripts/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/sstephenson/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/spec/rails_app/app/assets/stylesheets/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any styles
+ * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
+ * file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/spec/rails_app/app/controllers/application_controller.rb

@@ -0,0 +1,5 @@
+class ApplicationController < ActionController::Base
+  # Prevent CSRF attacks by raising an exception.
+  # For APIs, you may want to use :null_session instead.
+  protect_from_forgery with: :exception
+end

data/spec/rails_app/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/spec/rails_app/app/views/layouts/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Dummy</title>
+  <%= stylesheet_link_tag    'application', media: 'all', 'data-turbolinks-track' => true %>
+  <%= javascript_include_tag 'application', 'data-turbolinks-track' => true %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/spec/rails_app/bin/bundle

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+load Gem.bin_path('bundler', 'bundle')

data/spec/rails_app/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path('../../config/application',  __FILE__)
+require_relative '../config/boot'
+require 'rails/commands'

data/spec/rails_app/bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative '../config/boot'
+require 'rake'
+Rake.application.run

data/spec/rails_app/config/application.rb

@@ -0,0 +1,29 @@
+require File.expand_path('../boot', __FILE__)
+
+# Pick the frameworks you want:
+require "active_record/railtie"
+require "action_controller/railtie"
+require "action_mailer/railtie"
+require "action_view/railtie"
+require "sprockets/railtie"
+# require "rails/test_unit/railtie"
+
+Bundler.require(*Rails.groups)
+require "rails_app"
+
+module Dummy
+  class Application < Rails::Application
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration should go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded.
+
+    # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
+    # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
+    # config.time_zone = 'Central Time (US & Canada)'
+
+    # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
+    # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
+    # config.i18n.default_locale = :de
+  end
+end
+

data/spec/rails_app/config/boot.rb

@@ -0,0 +1,5 @@
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../../Gemfile', __FILE__)
+
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+$LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__)

data/spec/rails_app/config/database.yml

@@ -0,0 +1,25 @@
+# SQLite version 3.x
+#   gem install sqlite3
+#
+#   Ensure the SQLite 3 gem is defined in your Gemfile
+#   gem 'sqlite3'
+#
+default: &default
+  adapter: sqlite3
+  pool: 5
+  timeout: 5000
+
+development:
+  <<: *default
+  database: db/development.sqlite3
+
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test:
+  <<: *default
+  database: db/test.sqlite3
+
+production:
+  <<: *default
+  database: db/production.sqlite3

data/spec/rails_app/config/environment.rb

@@ -0,0 +1,5 @@
+# Load the Rails application.
+require File.expand_path('../application', __FILE__)
+
+# Initialize the Rails application.
+Rails.application.initialize!