duple 0.0.1

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+scratch
+config

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/.simplecov

@@ -0,0 +1,14 @@
+class SimpleCov::Formatter::QualityFormatter
+  def format(result)
+    SimpleCov::Formatter::HTMLFormatter.new.format(result)
+    File.open('coverage/covered_percent', 'w') do |f|
+      f.puts result.source_files.covered_percent.to_f
+    end
+  end
+end
+SimpleCov.formatter = SimpleCov::Formatter::QualityFormatter
+
+SimpleCov.start do
+  add_filter '/spec/'
+  add_group 'CLI', 'lib/duple/cli'
+end

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in duple.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Jason Wadsworth
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# Duple
+
+Duple makes it easy to move PostgreSQL data around between your deployment
+environments. Duple knows how to move data from one heroku environment to
+another and how to load it into your local database. It can execute rake or
+Heroku commands before and after your loading data into your target
+environment. This is great for scrubbing, replacing, trimming or generating
+data for your test environments.
+
+## Installation
+
+Install the gem:
+
+    $ gem install duple
+
+Generate a config file:
+
+    $ duple init
+
+## Configuration
+
+The generated config file contains samples of all the different ways you can
+figure the application. Read and modify it, or clear it out and write your own.
+
+## Usage
+
+    # Resets the stage database
+    # Loads the latest production snapshot into the stage database
+    $ duple refresh production stage
+
+    # Downloads the latest full snapshot from production
+    # Resets the development database
+    # Loads the snapshot into the development database
+    $ duple refresh production development
+
+    # Captures a new production database snapshot
+    # Downloads the latest full snapshot from production
+    # Resets the development database
+    # Loads the snapshot into the development database
+    $ duple refresh production development --capture
+
+    # Downloads the schema and a subset of data from stage
+    # Resets the backstage database
+    # Loads the structure and the subset into the backstage database
+    $ duple refresh stage backstage --group minimal
+
+    # Downloads the data from the specified tables from the stage database
+    # Loads the data into the backstage database
+    $ duple copy stage backstage --tables products categories
+
+## Future
+
+  * Everything.
+  * Support for skipping pre- and post- refresh steps.
+  * Support for running pre- and post- refresh steps by themselves.
+  * Support for other data stores.
+
+## Contributing
+
+  1. Fork it
+  2. Create your feature branch (`git checkout -b my-new-feature`)
+  3. Commit your changes (`git commit -am 'Add some feature'`)
+  4. Push to the branch (`git push origin my-new-feature`)
+  5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,21 @@
+#!/usr/bin/env rake
+
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+begin
+  require 'cane/rake_task'
+
+  desc "Run cane to check quality metrics"
+  Cane::RakeTask.new(:cane) do |cane|
+    cane.style_measure = 100
+    cane.style_glob = '{lib}/**/*.rb'
+    cane.gte = {'coverage/covered_percent' => 95}
+  end
+rescue LoadError
+  warn "cane not available, quality task not provided."
+end
+
+task :default => [:spec, :cane]

data/bin/duple

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "thor"
+
+if File.exists?(File.join(File.expand_path('../..', __FILE__), '.git'))
+  lib_path = File.expand_path('../../lib', __FILE__)
+  $:.unshift(lib_path)
+end
+
+require 'duple'
+Duple::CLI::Root.start

data/duple.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'duple/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = 'duple'
+  gem.version       = Duple::VERSION
+  gem.authors       = ['Jason Wadsworth']
+  gem.email         = ['jdwadsworth@gmail.com']
+  gem.description   = %q{
+    Duple simplifies moving and processing data snapshots between development,
+    testing and production environements.
+  }
+  gem.summary       = %q{Moves PostgreSQL data from here to there.}
+  gem.homepage      = 'https://github.com/subakva/duple'
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_dependency('thor')
+  gem.add_development_dependency('rake', ['~> 0.9.2'])
+  gem.add_development_dependency('rspec', ['~> 2.11.0'])
+  gem.add_development_dependency('rspec-fire', ['~> 1.1.3'])
+  gem.add_development_dependency('cane', ['~> 2.3.0'])
+  gem.add_development_dependency('simplecov', ['~> 0.7.1'])
+
+end

data/lib/duple.rb

@@ -0,0 +1,3 @@
+require 'duple/version'
+require 'duple/configuration'
+require 'duple/cli/root'

data/lib/duple/cli/config.rb

@@ -0,0 +1,63 @@
+module Duple
+  module CLI
+    class Config < Thor
+      include Duple::CLI::Helpers
+
+      config_option
+
+      no_tasks do
+        def print_hash(header, values)
+          say header
+          say '-' * 80
+          print_table values, indent: 2
+          say
+        end
+
+        def print_tasks(header, task_list)
+          say header
+          say '-' * 80
+          task_list.each do |task_name, commands|
+            say
+            say '  ' + task_name
+            print_table commands, indent: 4
+          end
+          say
+        end
+      end
+
+      desc 'environments', 'Prints the environment configurations.'
+      def environments
+        print_hash('Environments', config.environments)
+      end
+
+      desc 'groups', 'Prints the group configurations.'
+      def groups
+        print_hash('Groups', config.groups)
+      end
+
+      desc 'pre-refresh', 'Prints the pre-refresh tasks.'
+      def pre_refresh
+        print_tasks('Pre-Refresh Tasks', config.pre_refresh_tasks)
+      end
+
+      desc 'post-refresh', 'Prints the post-refresh tasks.'
+      def post_refresh
+        print_tasks('Post-Refresh Tasks', config.post_refresh_tasks)
+      end
+
+      desc 'other', 'Prints other options.'
+      def other
+        print_hash('Other Options', config.other_options)
+      end
+
+      desc 'all', 'Prints the current configuration.'
+      def all
+        environments
+        groups
+        pre_refresh
+        post_refresh
+        other
+      end
+    end
+  end
+end

data/lib/duple/cli/copy.rb

@@ -0,0 +1,29 @@
+module Duple
+  module CLI
+    class Copy < Thor::Group
+      include Duple::CLI::Helpers
+
+      config_option
+      source_option
+      target_option
+      group_option
+      capture_option
+      dry_run_option
+      tables_option
+
+      def require_included_tables
+        unless config.included_tables.size > 0
+          raise ArgumentError.new('One of --group or --tables options is required.')
+        end
+      end
+
+      def dump_data
+        postgres.pg_dump(dump_flags, data_file_path, source_credentials)
+      end
+
+      def restore_data
+        postgres.pg_restore('-e -v --no-acl -O -a', data_file_path, target_credentials)
+      end
+    end
+  end
+end

data/lib/duple/cli/helpers.rb

@@ -0,0 +1,192 @@
+require 'yaml'
+
+module Duple
+  module CLI
+    module Helpers
+
+      def self.included(base)
+        base.send(:include, Thor::Actions)
+        base.send(:include, InstanceMethods)
+        base.send(:extend, ClassMethods)
+      end
+
+      module ClassMethods
+        def source_root
+          File.expand_path(File.join(File.dirname(__FILE__), '../../..'))
+        end
+
+        def config_option
+          class_option :config,
+            desc: 'The location of the config file.',
+            type: :string,
+            aliases: '-c'
+        end
+
+        def source_option
+          class_option :source,
+            desc: 'The name of the source environment.',
+            type: :string,
+            aliases: '-s'
+        end
+
+        def target_option
+          class_option :target,
+            desc: 'The name of the target environment.',
+            type: :string,
+            aliases: '-t'
+        end
+
+        def group_option
+          class_option :group,
+            desc: 'Name of the group configuration to use when dumping source data.',
+            type: :string,
+            aliases: '-g'
+        end
+
+        def capture_option
+          class_option :capture,
+            desc: 'Capture a new source snapshot before refreshing.',
+            type: :boolean
+        end
+
+        def dry_run_option
+          class_option :dry_run,
+            desc: 'Perform a dry run of the command. No data will be moved.',
+            type: :boolean,
+            aliases: '--dry-run'
+        end
+
+        def tables_option(opts = nil)
+          opts ||= {}
+          opts = {
+            desc:     'A list of tables to include when dumping source data.',
+            required: false,
+            type:     :array,
+            aliases:  '-t'
+          }.merge(opts)
+          class_option :tables, opts
+        end
+      end
+
+      module InstanceMethods
+        def default_config_path
+          File.join('config', 'duple.yml')
+        end
+
+        def app_config_path(verify_file = true)
+          config_path = options[:config] || default_config_path
+          if verify_file && !File.exists?(config_path)
+            raise ArgumentError.new("Missing config file: #{config_path}")
+          end
+          config_path
+        end
+
+        def runner
+          @runner ||= Duple::Runner.new(dry_run: config.dry_run?)
+        end
+
+        def postgres
+          @pg_runner ||= Duple::PGRunner.new(runner)
+        end
+
+        def heroku
+          @heroku ||= Duple::HerokuRunner.new(runner)
+        end
+
+        def source_appname
+          @source_appname ||= config.heroku_name(config.source_environment)
+        end
+
+        def target_appname
+          @target_appname ||= config.heroku_name(config.target_environment)
+        end
+
+        def dump_dir_path
+          File.join('tmp', 'duple')
+        end
+
+        def data_file_path
+          @data_file_path ||= File.join(dump_dir_path, "#{config.source_name}-data.dump")
+        end
+
+        def structure_file_path
+          filename = "#{config.source_name}-structure.dump"
+          @structure_file_path ||= File.join(dump_dir_path, filename)
+        end
+
+        def snapshot_file_path(timestamp)
+          filename = "#{config.source_name}-#{timestamp}.dump"
+          @structure_file_path ||= File.join(dump_dir_path, filename)
+        end
+
+        def fetch_heroku_credentials(appname)
+          config_vars = heroku.capture(appname, "config")
+
+          return config.dry_run_credentials(appname) if config.dry_run?
+
+          db_url = config_vars.split("\n").detect { |l| l =~ /DATABASE_URL/ }
+          raise ArgumentError.new("Missing DATABASE_URL variable for #{appname}") if db_url.nil?
+
+          db_url.match(
+            /postgres:\/\/(?<user>.*):(?<password>.*)@(?<host>.*):(?<port>\d*)\/(?<db>.*)/
+          )
+        end
+
+        def fetch_latest_snapshot_time(appname)
+          response = heroku.capture(appname, 'pgbackups')
+          last_line = response.split("\n").last
+          timestring = last_line.match(/\w+\s+(?<timestamp>[\d\s\/\:\.]+)\s+.*/)[:timestamp]
+          DateTime.strptime(timestring, '%Y/%m/%d %H:%M.%S')
+        end
+
+        def reset_database(env)
+          if config.heroku?(env)
+            appname = config.heroku_name(env)
+            heroku.run(appname, 'pg:reset')
+          else
+            # if yes?("Are you sure you want to reset the #{config.target_name} database?", :red)
+            runner.run('bundle exec rake db:drop db:create')
+          end
+        end
+
+        def dump_flags
+          include_tables = config.included_tables
+          include_flags = include_tables.map { |t| "-t #{t}" }
+
+          exclude_tables = config.excluded_tables
+          exclude_flags = exclude_tables.map { |t| "-T #{t}" }
+
+          flags = [ '-Fc -a', include_flags, exclude_flags ].flatten.compact.join(' ')
+        end
+
+        def source_credentials
+          @source_credentials ||= if config.heroku_source?
+                                    fetch_heroku_credentials(source_appname)
+                                  else
+                                    config.local_credentials
+                                  end
+        end
+
+        def target_credentials
+          @target_credentials ||= if config.heroku_target?
+                                    fetch_heroku_credentials(target_appname)
+                                  else
+                                    config.local_credentials
+                                  end
+        end
+
+        def config
+          @config ||= parse_config
+        end
+
+        def parse_config
+          config_path = File.join(destination_root, app_config_path)
+          config_data = File.read(config_path)
+          erbed = ERB.new(config_data).result
+          config_hash = YAML.load(erbed) || {}
+          Duple::Configuration.new(config_hash, options)
+        end
+      end
+    end
+  end
+end