duple 0.0.1

data/lib/duple/cli/init.rb

@@ -0,0 +1,15 @@
+module Duple
+  module CLI
+    class Init < Thor::Group
+      include Duple::CLI::Helpers
+
+      config_option
+
+      def create_sample_config
+        config_path = app_config_path(false)
+        empty_directory(File.dirname(config_path))
+        copy_file('templates/config/duple.yml', config_path)
+      end
+    end
+  end
+end

data/lib/duple/cli/refresh.rb

@@ -0,0 +1,75 @@
+module Duple
+  module CLI
+    class Refresh < Thor::Group
+      include Duple::CLI::Helpers
+
+      config_option
+      source_option
+      target_option
+      group_option
+      capture_option
+      tables_option
+
+      no_tasks do
+        def capture_snapshot?
+          config.capture? && config.heroku_source?
+        end
+
+        def fetch_snapshot_url?
+          config.heroku_source? && !config.filtered_tables?
+        end
+
+        def download_snapshot?
+          config.heroku_source? && config.local_target? && !config.filtered_tables?
+        end
+
+        def dump_data?
+          config.local_source? || config.filtered_tables?
+        end
+      end
+
+      def capture_snapshot
+        return unless capture_snapshot?
+
+        heroku.run(source_appname, 'pgbackups:capture')
+      end
+
+      def fetch_snapshot_url
+        return unless fetch_snapshot_url?
+
+        @source_snapshot_url = heroku.capture(source_appname, 'pgbackups:url').strip
+      end
+
+      def download_snapshot
+        return unless download_snapshot?
+
+        timestamp = fetch_latest_snapshot_time(source_appname)
+
+        @snapshot_path = snapshot_file_path(timestamp.strftime('%Y-%m-%d-%H-%M-%S'))
+        unless File.exists?(@snapshot_path)
+          runner.run("curl -o #{@snapshot_path} #{@source_snapshot_url}")
+        end
+      end
+
+      def dump_data
+        return unless dump_data?
+
+        postgres.pg_dump(dump_flags, data_file_path, source_credentials)
+      end
+
+      def reset_target
+        reset_database(config.target_environment)
+      end
+
+      def restore_database
+        if download_snapshot?
+          postgres.pg_restore('-e -v --no-acl -O -a', @snapshot_path, target_credentials)
+        elsif dump_data?
+          postgres.pg_restore('-e -v --no-acl -O -a', data_file_path, target_credentials)
+        else
+          heroku.run(target_appname, "pgbackups:restore DATABASE #{@source_snapshot_url}")
+        end
+      end
+    end
+  end
+end

data/lib/duple/cli/root.rb

@@ -0,0 +1,41 @@
+require 'thor'
+require 'thor/group'
+require 'duple/cli/helpers'
+require 'duple/cli/init'
+require 'duple/cli/copy'
+require 'duple/cli/config'
+require 'duple/cli/structure'
+require 'duple/cli/refresh'
+require 'duple/runner'
+require 'duple/pg_runner'
+require 'duple/heroku_runner'
+
+module Duple
+  module CLI
+    class Root < Thor
+      include Duple::CLI::Helpers
+
+      # HACK Override register to handle class_options for groups properly.
+      def self.register(klass, task_name, description)
+        super(klass, task_name, task_name, description)
+        tasks[task_name].options = klass.class_options
+      end
+
+      register Duple::CLI::Init,     'init',
+        'Generates a sample configuration file.'
+
+      register Duple::CLI::Copy,     'copy',
+        'Copies data from a source to a target database.'
+
+      register Duple::CLI::Structure,     'structure',
+        'Copies structure from a source to a target database.'
+
+      register Duple::CLI::Refresh,  'refresh',
+        'Resets and copies schema and data from a source to a target database'
+
+      register Duple::CLI::Config,
+        'config [COMMAND]', 'Manage your configuration.'
+    end
+  end
+end
+

data/lib/duple/cli/structure.rb

@@ -0,0 +1,23 @@
+module Duple
+  module CLI
+    class Structure < Thor::Group
+      include Duple::CLI::Helpers
+
+      config_option
+      source_option
+      target_option
+
+      def dump_structure
+        postgres.pg_dump('-Fc --no-acl -O -s', structure_file_path, source_credentials)
+      end
+
+      def reset_target
+        reset_database(config.target_environment)
+      end
+
+      def load_structure
+        postgres.pg_restore('-v --no-acl -O -s', structure_file_path, target_credentials)
+      end
+    end
+  end
+end

data/lib/duple/configuration.rb

@@ -0,0 +1,203 @@
+module Duple
+
+  # Represents the configuration that will be used to perform the data
+  # operations.
+  #
+  # This class should be the only place in the system that knows about the
+  # structure of the config file.
+  #
+  # This class should not have any knowledge of any particular database
+  # system. For example, this class can know about the concept of a "tables",
+  # but it should know nothing about flags for PostgreSQL commands.
+  class Configuration
+    HEROKU  = 'heroku'
+    LOCAL   = 'local'
+    VALID_TYPES = [HEROKU, LOCAL]
+
+    attr_reader :options, :raw_config
+
+    def initialize(config_hash, options)
+      @raw_config = config_hash
+      @options = options
+    end
+
+    def default_target_name
+      env_names_by_flag('default_target', true).first
+    end
+
+    def target_name
+      options[:target] || default_target_name
+    end
+
+    def target_environment
+      invalid_target_names = env_names_by_flag('allow_target', false, true)
+      if invalid_target_names.include?(target_name)
+        raise ArgumentError.new("Invalid target: #{target_name} is not allowed to be a target.")
+      end
+      environment(target_name)
+    end
+
+    def heroku_target?
+      heroku?(target_environment)
+    end
+
+    def local_target?
+      local?(target_environment)
+    end
+
+    def default_source_name
+      env_names_by_flag('default_source', true).first
+    end
+
+    def source_name
+      options[:source] || default_source_name
+    end
+
+    def source_environment
+      environment(source_name)
+    end
+
+    def heroku_source?
+      heroku?(source_environment)
+    end
+
+    def local_source?
+      local?(source_environment)
+    end
+
+    def heroku?(env)
+      env['type'] == Duple::Configuration::HEROKU
+    end
+
+    def local?(env)
+      env['type'] == Duple::Configuration::LOCAL
+    end
+
+    def heroku_name(env)
+      env['appname']
+    end
+
+    def dry_run?
+      options[:dry_run]
+    end
+
+    def capture?
+      options[:capture]
+    end
+
+    def group_name
+      options[:group]
+    end
+
+    def table_names
+      options[:tables] || []
+    end
+
+    def filtered_tables?
+      included_tables.size > 0 || excluded_tables.size > 0
+    end
+
+    # Returns an array of tables to include, based on the group config and the
+    # --tables option. An empty array indicates that ALL tables should be
+    # included. If the group has the include_all flag, an empty array will be
+    # returned.
+    def included_tables
+      tables = table_names
+      if group_name
+        g = group(group_name)
+        return [] if g['include_all']
+        tables += (g['include_tables'] || [])
+      end
+      tables.uniq.sort
+    end
+
+    # Returns an array of tables to exclude, based on the group config and the
+    # --tables option. The --tables option takes precedence over the --group
+    # option, so if a table is excluded from a group, but specified in the
+    # --tables option the table will NOT be excluded.
+     def excluded_tables
+      tables = []
+      if group_name
+        g = group(group_name)
+        tables += (g['exclude_tables'] || [])
+      end
+      tables -= table_names
+      tables
+    end
+
+    def dry_run_credentials(envname)
+      {
+        user:     "[#{envname}.USER]",
+        password: "[#{envname}.PASS]",
+        host:     "[#{envname}.HOST]",
+        port:     "[#{envname}.PORT]",
+        db:       "[#{envname}.DB]"
+      }
+    end
+
+    def local_credentials
+      # TODO: Override these defaults from the config
+      {
+        user:     'postgres',
+        password: '',
+        host:     'localhost',
+        port:     '5432',
+        db:       'duple_development'
+      }
+    end
+
+    def environments
+      raw_config['environments'] || {}
+    end
+
+    def environment(env_name)
+      env = environments[env_name]
+      raise ArgumentError.new("Invalid environment: #{env_name}") if env.nil?
+      env
+    end
+
+    def groups
+      raw_config['groups'] || {}
+    end
+
+    def group(group_name)
+      group = groups[group_name]
+      raise ArgumentError.new("Invalid group: #{group_name}") if group.nil?
+      group
+    end
+
+    def pre_refresh_tasks
+      raw_config['pre_refresh'] || {}
+    end
+
+    def pre_refresh_task(task_name)
+      task = pre_refresh_tasks[task_name]
+      raise ArgumentError.new("Invalid pre_refresh task: #{task_name}") if task.nil?
+      task
+    end
+
+    def post_refresh_tasks
+      raw_config['post_refresh'] || {}
+    end
+
+    def post_refresh_task(task_name)
+      task = post_refresh_tasks[task_name]
+      raise ArgumentError.new("Invalid post_refresh task: #{task_name}") if task.nil?
+      task
+    end
+
+    def other_options
+      raw_config.reject { |k,v| %w{environments groups pre_refresh post_refresh}.include?(k) }
+    end
+
+    protected
+
+    def env_names_by_flag(flag_name, flag_value, allow_multiple = false)
+      matching_envs = environments.select { |n, c| c[flag_name] == flag_value }
+      if matching_envs.size > 1 && !allow_multiple
+        raise ArgumentError.new("Only a single environment can be #{flag_name}.")
+      end
+      matching_envs.keys
+    end
+  end
+end

data/lib/duple/heroku_runner.rb

@@ -0,0 +1,25 @@
+module Duple
+  class HerokuRunner
+    def initialize(runner)
+      @runner = runner
+    end
+
+    def run(appname, args, tail = nil)
+      @runner.run(heroku_command(appname, args, tail))
+    end
+
+    def capture(appname, args, tail = nil)
+      @runner.capture(heroku_command(appname, args, tail))
+    end
+
+    protected
+
+    def heroku_command(appname, args, tail = nil)
+      command = []
+      command << %{heroku #{args}}
+      command << %{-a #{appname}}
+      command << %{tail} if tail
+      command.join(' ')
+    end
+  end
+end

data/lib/duple/pg_runner.rb

@@ -0,0 +1,27 @@
+module Duple
+  class PGRunner
+    def initialize(runner)
+      @runner = runner
+    end
+
+    def pg_dump(flags, dump_file, credentials)
+      pg_command('pg_dump', flags, credentials, nil, "> #{dump_file}")
+    end
+
+    def pg_restore(flags, dump_file, credentials)
+      pg_command('pg_restore', flags, credentials, '-d', "< #{dump_file}")
+    end
+
+    def pg_command(pg_command, flags, credentials, db_flag, tail = nil)
+        command = []
+        command << %{PGPASSWORD="#{credentials[:password]}"}
+        command << pg_command
+        command << flags
+        command << %{-h #{credentials[:host]} -U #{credentials[:user]} -p #{credentials[:port]}}
+        command << db_flag if db_flag
+        command << credentials[:db]
+        command << tail if tail
+        @runner.run(command.join(' '))
+    end
+  end
+end

data/lib/duple/runner.rb

@@ -0,0 +1,63 @@
+require 'fileutils'
+
+module Duple
+  class Runner
+
+    def initialize(options = nil)
+      options ||= {}
+      @options = {
+        log: STDOUT,
+        log_format: ' * Running: %s',
+        dry_run: false,
+        recorder: nil
+      }.merge(options)
+
+      if recorder? && !valid_recorder?
+        raise ArgumentError.new("Invalid :recorder option: #{@options[:recorder]}")
+      end
+    end
+
+    def run(command, capture = false)
+      log_command(command)
+      record_command(command) if recorder?
+
+      return if dry_run?
+
+      result = capture ? `#{command}` : system(command)
+      raise RuntimeError.new("Command failed: #{$?}") unless $?.success?
+      result
+    end
+
+    def capture(command)
+      run(command, true)
+    end
+
+    def record_command(command)
+      @options[:recorder].puts command
+    end
+
+    def log_command(command)
+      return unless @options[:log]
+
+      formatted = @options[:log_format] % command
+      @options[:log].puts formatted
+    end
+
+    def valid_recorder?
+      @options[:recorder].respond_to?(:puts)
+    end
+
+    def recorder?
+      @options[:recorder]
+    end
+
+    def live?
+      !@options[:dry_run]
+    end
+
+    def dry_run?
+      @options[:dry_run]
+    end
+  end
+end
+