data-exporter 1.3.7

data/lib/data_exporter/configuration.rb

@@ -0,0 +1,164 @@
+require 'erb'
+require 'yaml'
+require 'active_support/core_ext/hash'
+
+module DataExporter
+  class Configuration
+    ALLOWED_OVERRIDES = %w(mode export_dir download_dir unpack_dir pii_file csv)
+
+    def initialize
+      reset!
+    end
+
+    def reset!
+      @config_file = nil
+      @override = HashWithIndifferentAccess.new
+      @sections = HashWithIndifferentAccess.new { |h,k| h[k] = HashWithIndifferentAccess.new }
+    end
+
+    def database=(h = {})
+      @sections[mode][:mysql] ||= HashWithIndifferentAccess.new
+      @sections[mode][:mysql].merge!(h)
+    end
+
+    def database
+      @sections[mode][:mysql] ||= HashWithIndifferentAccess.new
+      @sections[mode][:mysql]
+    end
+    alias :mysql :database
+
+    def mysqldump_path
+      @sections[mode].fetch(:mysqldump_path, 'mysqldump')
+    end
+
+    def mysqldump_options
+      @sections[mode].fetch(:mysqldump_options, '').split(/\s+/).compact
+    end
+
+    def s3
+      @sections[mode][:s3] ||= HashWithIndifferentAccess.new
+      @sections[mode][:s3].tap do |s3|
+        s3[:bucket_name].sub!(%r{\As3://}, '') if s3[:bucket_name]
+        s3[:bucket] = s3[:bucket_name]
+      end
+    end
+
+    def sftp
+      @sections[mode][:sftp] ||= HashWithIndifferentAccess.new
+      @sections[mode][:sftp].tap do |sftp|
+        sftp[:user] ||= ENV['USER']
+      end
+    end
+
+    def redis
+      @sections[mode][:redis] ||= HashWithIndifferentAccess.new
+      @sections[mode][:redis]
+    end
+
+    def mode
+      @override.fetch(:mode, :default)
+    end
+
+    %w(export_dir download_dir unpack_dir).each do |dir|
+      define_method(dir) do
+        @override[dir] || @sections[mode].fetch(dir, Dir.mktmpdir('data_exporter'))
+      end
+    end
+
+    def backup_dir
+      @sections[mode][:backup_dir]
+    end
+
+    def backup_key
+      absolute_config_path(@sections[mode][:backup_key])
+    end
+
+    def backup_prefix
+      @sections[mode][:backup_prefix] || @sections[mode][:s3][:prefix]
+    end
+
+    def archive_base_directory
+      @sections[mode].fetch(:archive_base_directory, 'data_exporter')
+    end
+
+    def pii_file
+      @override[:pii_file] || @sections[mode][:pii_file]
+    end
+
+    def pii_fields(table)
+      return [] unless pii_file
+      @pii_data ||= load_yaml_file(pii_file)
+      @pii_data.fetch(table, []).map(&:to_s)
+    end
+
+    def csv_enabled?
+      @override[:csv] || @sections[mode][:export_csv]
+    end
+
+    def sftp_enabled?
+      sftp && sftp[:host] && sftp[:user]
+    end
+
+    def load(options = {})
+      @config_file = options[:config_file]
+      @override.merge!(options.select { |k,_| ALLOWED_OVERRIDES.include?(k.to_sym) || ALLOWED_OVERRIDES.include?(k.to_s) })
+      process_configuration(load_yaml_file(@config_file))
+      validate!(options)
+      self
+    end
+
+    def validate!(options = {})
+      raise ArgumentError, "#{@config_file} missing s3 section" unless s3_config_complete? unless sftp_enabled?
+      raise ArgumentError, "#{@config_file} missing mysql section" unless database_config_complete? if options[:mysql_required]
+      raise ArgumentError, "#{@config_file} missing redis section" unless redis_config_complete? if options[:redis_key_prefix]
+    end
+
+    private
+
+    def config_path
+      File.expand_path('..', @config_file)
+    end
+
+    def absolute_config_path(path)
+      if File.absolute_path(path) == path
+        path
+      else
+        File.join(config_path, path)
+      end
+    end
+
+    def load_yaml_file(yaml_file)
+      case yaml_file
+      when '-'
+        YAML.load(STDIN.read)
+      when /yml.erb\z/
+        YAML.load(ERB.new(File.read(yaml_file)).result)
+      else
+        YAML.load_file(yaml_file)
+      end.with_indifferent_access
+    end
+
+    def process_configuration(config_hash)
+      return process_configuration_with_sections(config_hash) if config_hash['default'].is_a?(Hash)
+      @sections[:default] = config_hash
+    end
+
+    def process_configuration_with_sections(config_hash)
+      config_hash.each do |section, config|
+        @sections[section] = config
+      end
+    end
+
+    def database_config_complete?
+      database && database[:username] && database[:database]
+    end
+
+    def s3_config_complete?
+      s3 && s3[:bucket] && s3[:access_key_id] && s3[:secret_access_key]
+    end
+
+    def redis_config_complete?
+      redis && redis[:host] && redis[:port]
+    end
+  end
+end

data/lib/data_exporter/version.rb

@@ -0,0 +1,3 @@
+module DataExporter
+  VERSION = '1.3.7'
+end

data/spec/actions_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+
+describe DataExporter::Actions do
+  let(:instance) do
+    Class.new do
+      include DataExporter::Actions
+    end.new
+  end
+
+  describe '#find_last_sftp_backup' do
+    let(:prefix) { 'data-export' }
+    let(:suffix) { 'sql.gz.enc' }
+
+    subject do
+      instance.find_last_sftp_backup(prefix, suffix)
+    end
+
+    context 'with --sftp' do
+      let(:remote_files) do
+        [
+          OpenStruct.new(name: '/data_export-2013-05-05_db.sql.gz.enc', mtime: 100, size: 10),
+          OpenStruct.new(name: '/data_export-2013-05-06_db.sql.gz.enc', mtime: 200, size: 20),
+          OpenStruct.new(name: '/data_export-2013-05-06_db.sql.gz.enc', mtime: 300, size: 10)
+        ]
+      end
+
+      before do
+        sftp_file_doubles = []
+        remote_files.each do |remote_file|
+          sftp_file_double = double
+          expect(sftp_file_double).to receive(:name).at_most(:once).and_return(remote_file.name)
+          expect(sftp_file_double).to receive(:attributes).at_least(:once).and_return(double(:mtime => remote_file.mtime, :size => remote_file.size))
+          sftp_file_doubles << sftp_file_double
+        end
+
+        sftp_dir_double = double
+        expect(sftp_dir_double).to receive(:glob).with('/', 'data-export*sql.gz.enc').
+          and_yield(sftp_file_doubles[2]).
+          and_yield(sftp_file_doubles[1]).
+          and_yield(sftp_file_doubles[0])
+
+        sftp_session_double = double
+        expect(sftp_session_double).to receive(:dir).and_return(sftp_dir_double)
+        expect(instance).to receive(:sftp).and_return(sftp_session_double)
+      end
+
+      it { is_expected.to eq(remote_files.last) }
+    end
+  end
+end

data/spec/cli_spec.rb

@@ -0,0 +1,430 @@
+require 'spec_helper'
+
+describe DataExporter::CLI do
+  include DataExporter::Actions
+
+  def stub_s3_upload(remote_file)
+    s3_object_double = double
+    expect(s3_object_double).to receive(:write).once
+
+    s3_objects_double = double('s3_objects')
+    expect(s3_objects_double).to receive(:[]).with(remote_file).and_return(s3_object_double)
+
+    expect_any_instance_of(AWS::S3::Bucket).to receive(:objects).and_return(s3_objects_double)
+  end
+
+  def stub_s3_find_last(remote_file)
+    s3_object_double = double('s3_object', :key => remote_file, :last_modified => Time.now.to_i, :content_length => 1024)
+
+    s3_objects_double = double('s3_objects')
+    expect(s3_objects_double).to receive(:with_prefix).and_return([s3_object_double])
+
+    expect_any_instance_of(AWS::S3::Bucket).to receive(:objects).and_return(s3_objects_double)
+  end
+
+  def stub_s3_download(local_file, remote_file)
+    s3_object_double = double('s3_object', :key => remote_file, :last_modified => Time.now.to_i, :content_length => 1024)
+    expect(s3_object_double).to receive(:read).and_yield(File.read(local_file))
+
+    s3_objects_double = double('s3_objects')
+    expect(s3_objects_double).to receive(:with_prefix).with(File.join(backup_dir, backup_prefix)).and_return([s3_object_double])
+
+    expect_any_instance_of(AWS::S3::Bucket).to receive(:objects).and_return(s3_objects_double)
+  end
+
+  let(:export_dir) { Dir.mktmpdir('export') }
+  let(:backup_dir) { 'backups' }
+  let(:unpack_dir) { Dir.mktmpdir('unpack') }
+  let(:download_dir) { Dir.mktmpdir('download') }
+  let(:backup_key) { File.expand_path('../fixtures/backup_key', __FILE__) }
+  let(:backup_prefix) { 'data_export' }
+  let(:archive_base_directory) { 'data_exporter' }
+  let(:sftp_host) { 'localhost' }
+  let(:sftp_user) { ENV['USER'] }
+  let(:archive_dir) { Dir.mktmpdir('archive') }
+
+  let(:mysql_config) do
+    {
+      adapter:  'mysql2',
+      host:     'localhost',
+      database: 'centurion_test',
+      username: 'root'
+    }
+  end
+
+  let(:s3_config) do
+    {
+      access_key_id: 'spec',
+      secret_access_key: 'spec',
+      bucket_name: 'spec',
+      prefix: backup_prefix
+    }
+  end
+
+  let(:redis_config) do
+    {
+      host: 'localhost',
+      port: 6379
+    }
+  end
+
+  let(:sftp_config) { {} }
+
+  let(:config_hash) do
+    {
+      export_dir: export_dir,
+      backup_dir: backup_dir,
+      backup_key: backup_key,
+      unpack_dir: unpack_dir,
+      download_dir: download_dir,
+      mysqldump_options: '--extended-insert --single-transaction',
+      mysql: mysql_config,
+      redis: redis_config,
+      s3: s3_config,
+      sftp: sftp_config
+    }
+  end
+
+  let(:pii_hash) do
+    {
+      :users => [:last_name, :email]
+    }
+  end
+
+  let(:config_file) do
+    File.open(File.join(Dir.mktmpdir('data_exporter'), 'config.yml'), 'w') do |file|
+      file.write config_hash.to_yaml
+      file.flush
+    end
+  end
+
+  let(:pii_file) do
+    File.open(File.join(Dir.mktmpdir('data_exporter'), 'pii.yml'), 'w') do |file|
+      file.write pii_hash.to_yaml
+      file.flush
+    end
+  end
+
+  before do
+    DataExporter.configure do |config|
+      config.reset!
+      config.database = {
+        adapter:  'mysql2',
+        host:     'localhost',
+        database: 'centurion_test',
+        username: 'root'
+      }
+    end
+  end
+
+  after do
+    FileUtils.rm(config_file)
+    FileUtils.rm(pii_file)
+    FileUtils.rm_rf(export_dir)
+    FileUtils.rm_rf(unpack_dir)
+    FileUtils.rm_rf(download_dir)
+  end
+
+  let(:archive_contents) { Dir.entries(unpack_dir) }
+
+  describe '#export_task' do
+    let(:options) { [] }
+
+    before do
+      Timecop.freeze('2013-04-20')
+    end
+
+    let(:encrypted_archive_base_name) { 'data_export_2013-04-20-00-00_db.sql.gz.enc' }
+    let(:encrypted_archive) { File.join(export_dir, encrypted_archive_base_name) }
+    let(:remote_encrypted_archive) { File.join(backup_dir, encrypted_archive_base_name) }
+
+    subject do
+      DataExporter::CLI.start(['export', *options, '--quiet', '--preserve', '--config-file', config_file.path])
+    end
+
+    shared_examples 'mysqldump export' do
+      it 'exports a sql file' do
+        expect(archive_contents.size).to eq(3)
+        expect(archive_contents.last).to match(/data_export.*_db.sql/)
+        backup_contents = File.read(File.join(unpack_dir, archive_contents.last)).split("\n")
+        expect(backup_contents[0]).to match(/\A-- MySQL dump/)
+      end
+    end
+
+    context 'with default options' do
+      before do
+        stub_s3_upload(remote_encrypted_archive)
+        subject
+        unpack(encrypted_archive, unpack_dir)
+      end
+
+      it_behaves_like 'mysqldump export'
+
+      context 'without config.backup_dir' do
+        let(:backup_dir) { nil }
+        let(:remote_encrypted_archive) { encrypted_archive_base_name }
+
+        it_behaves_like 'mysqldump export'
+      end
+
+      context 'with --csv option' do
+        let(:options) { ['--csv'] }
+
+        let(:encrypted_archive_base_name) { 'data_export_2013-04-20-00-00_db.csv.tar.gz.enc' }
+
+        it 'exports an archive of exported csv files' do
+          expect(archive_contents).to include 'users_1.csv'
+          expect(archive_contents).to_not include 'schema_migrations_1.csv'
+          expect(archive_contents).to_not include 'checksums_1.csv'
+        end
+
+        it 'contains a user export csv file' do
+          backup_contents = File.read(File.join(unpack_dir, 'users_1.csv')).split("\n")
+          expect(backup_contents[0]).to eq("id,username,first_name,last_name,email,created_at,updated_at")
+          expect(backup_contents[1]).to match(/\A1,,Emily,James,emily@socialcast\.com.*/)
+          expect(backup_contents[2]).to match(/\A2,,Jennifer,Lawson,jennifer@socialcast\.com.*/)
+        end
+      end
+
+      context 'with --csv and --pii-file option' do
+        let(:options) { ['--csv', '--pii-file', pii_file] }
+
+        let(:encrypted_archive_base_name) { 'data_export_2013-04-20-00-00_db.csv.tar.gz.enc' }
+
+        it 'exports an archive of exported csv files' do
+          expect(archive_contents).to include 'users_1.csv'
+          expect(archive_contents).to_not include 'schema_migrations_1.csv'
+          expect(archive_contents).to_not include 'checksums_1.csv'
+        end
+
+        it 'contains a user export csv file without PII fields' do
+          backup_contents = File.read(File.join(unpack_dir, 'users_1.csv')).split("\n")
+          expect(backup_contents[0]).to eq("id,username,first_name,created_at,updated_at")
+          expect(backup_contents[1]).to match(/\A1,,Emily,.*/)
+          expect(backup_contents[1]).not_to match(/emily@socialcast\.com/)
+          expect(backup_contents[2]).to match(/\A2,,Jennifer,.*/)
+          expect(backup_contents[2]).not_to match(/jennifer@socialcast\.com/)
+        end
+      end
+    end
+
+    context 'with --csv and --archive-dir options' do
+      let(:mysql_config) { {} }
+      let(:options) { ['--csv', '--archive-dir', archive_dir, '--date', '2014-04-20'] }
+      let(:contents) do
+        <<-EOS
+        id,username,first_name,last_name,email
+        3,bot,Production,Bot,bot@socialcast.com
+        EOS
+      end
+
+      before do
+        stub_s3_upload(remote_encrypted_archive)
+        File.open(File.join(archive_dir, 'users_1.csv'), 'w') do |file|
+          file.write contents
+        end
+        subject
+        unpack(encrypted_archive, unpack_dir)
+      end
+
+      let(:encrypted_archive_base_name) { 'data_export_2014-04-20-00-00_db.csv.tar.gz.enc' }
+
+      it 'exports an archive of exported csv files' do
+        expect(archive_contents).to include 'users_1.csv'
+        expect(archive_contents).to_not include 'schema_migrations_1.csv'
+        expect(archive_contents).to_not include 'checksums_1.csv'
+      end
+
+      it 'contains a user export csv file' do
+        backup_contents = File.read(File.join(unpack_dir, 'users_1.csv'))
+        expect(backup_contents).to eq(contents)
+      end
+    end
+
+    context 'with sftp config' do
+      let(:sftp_config) do
+        {
+          host: sftp_host,
+          user: sftp_user,
+          backup_dir: backup_dir
+        }
+      end
+
+      before do
+        expect_any_instance_of(AWS::S3::S3Object).to receive(:write).never
+        sftp_double = double
+        expect(sftp_double).to receive(:mkdir).with(backup_dir)
+        expect(sftp_double).to receive(:upload!).with(encrypted_archive, File.join(backup_dir, encrypted_archive_base_name))
+        expect(Net::SFTP).to receive(:start).with(sftp_host, sftp_user, {timeout: 30}).and_return(sftp_double)
+        subject
+      end
+
+      it 'uploads the archive via SFTP' do
+      end
+    end
+
+    context 'with an incomplete mysql configuration section' do
+      let(:mysql_config) { {} }
+
+      it { expect { subject }.to raise_error ArgumentError, /missing mysql section/ }
+    end
+  end
+
+  describe '#unpack_task' do
+    let(:options) { [] }
+    let(:encrypted_archive_basename) { 'data_export-2013-05-05_db.sql.gz.enc' }
+    let(:encrypted_archive) { File.join(export_dir, 'data_export-2013-05-05_db.sql.gz.enc') }
+    let(:remote_encrypted_archive) { File.join(backup_dir, 'data_export-2013-05-05_db.sql.gz.enc') }
+
+    subject do
+      DataExporter::CLI.start(['unpack', *options, '--quiet', '--config-file', config_file.path])
+    end
+
+    context 'when backups are found' do
+      before do
+        export(backup_key, encrypted_archive)
+        stub_s3_download(encrypted_archive, remote_encrypted_archive)
+        subject
+      end
+
+      it 'unpacks a sql file' do
+        expect(archive_contents.size).to eq(3)
+        expect(archive_contents.last).to match(/data_export.*_db.sql/)
+        backup_contents = File.read(File.join(unpack_dir, archive_contents.last)).split("\n")
+        expect(backup_contents[0]).to match(/\A-- MySQL dump/)
+      end
+
+      context 'with --csv option' do
+        let(:options) { ['--csv'] }
+        let(:encrypted_archive) { File.join(export_dir, 'data_export-2013-05-05_db.csv.tar.gz.enc') }
+        let(:remote_encrypted_archive) { File.join(backup_dir, 'data_export-2013-05-05_db.csv.tar.gz.enc') }
+
+        it 'unpacks an archive of exported csv files' do
+          expect(archive_contents).to include 'users_1.csv'
+          expect(archive_contents).to_not include 'schema_migrations_1.csv'
+          expect(archive_contents).to_not include 'checksums_1.csv'
+        end
+
+        it 'contains a user export csv file' do
+          backup_contents = File.read(File.join(unpack_dir, 'users_1.csv')).split("\n")
+          expect(backup_contents[0]).to eq("id,username,first_name,last_name,email,created_at,updated_at")
+          expect(backup_contents[1]).to match(/\A1,,Emily,James,emily@socialcast\.com.*/)
+          expect(backup_contents[2]).to match(/\A2,,Jennifer,Lawson,jennifer@socialcast\.com.*/)
+        end
+      end
+
+      context 'with --date option' do
+        let(:options) { ['--date', '2013-05-05'] }
+
+        it 'unpacks a sql file' do
+          expect(archive_contents.size).to eq(3)
+          expect(archive_contents.last).to match(/data_export.*_db.sql/)
+          backup_contents = File.read(File.join(unpack_dir, archive_contents.last)).split("\n")
+          expect(backup_contents[0]).to match(/\A-- MySQL dump/)
+        end
+      end
+    end
+
+    context 'when Open3.pipeline returns a non-zero exit status' do
+      before do
+        export(backup_key, encrypted_archive)
+        stub_s3_download(encrypted_archive, remote_encrypted_archive)
+        allow(Open3).to receive(:pipeline_start).and_yield([double(value: double(success?: true)), double(value: double(success?: false))])
+      end
+
+      it { expect { subject }.to raise_error SystemExit, /Problem unpacking/ }
+
+      context 'with --csv option' do
+        let(:options) { ['--csv'] }
+        let(:encrypted_archive) { File.join(export_dir, 'data_export-2013-05-05_db.csv.tar.gz.enc') }
+        let(:remote_encrypted_archive) { File.join(backup_dir, 'data_export-2013-05-05_db.csv.tar.gz.enc') }
+        it { expect { subject }.to raise_error SystemExit, /Problem unpacking/ }
+      end
+    end
+
+    context 'when backups are not found' do
+      before do
+        export(backup_key, encrypted_archive)
+        s3_objects_double = double('s3_objects')
+        expect(s3_objects_double).to receive(:with_prefix)
+        expect_any_instance_of(AWS::S3::Bucket).to receive(:objects).and_return(s3_objects_double)
+      end
+
+      it { expect { subject }.to raise_error SystemExit, /No backups found/ }
+    end
+
+    context 'with sftp config' do
+      let(:sftp_config) do
+        {
+          host: sftp_host,
+          user: sftp_user
+        }
+      end
+
+      let(:remote_encrypted_archive) { File.join(backup_dir, 'data_export-2013-05-05_db.sql.gz.enc') }
+      let(:downloaded_encrypted_archive) { File.join(download_dir, 'data_export-2013-05-05_db.sql.gz.enc') }
+
+      context 'when backups exist' do
+        before do
+          export(backup_key, encrypted_archive)
+
+          sftp_dir_double, sftp_file_double, sftp_session_double = double, double, double
+          expect(sftp_file_double).to receive(:name).and_return(encrypted_archive_basename)
+          expect(sftp_file_double).to receive(:attributes).at_least(:once).and_return(double(:mtime => Time.now.to_i, :size => nil))
+          expect(sftp_dir_double).to receive(:glob).with(backup_dir, 'data_export*sql.gz.enc').and_yield(sftp_file_double)
+          expect(sftp_session_double).to receive(:dir).and_return(sftp_dir_double)
+          expect(sftp_session_double).to receive(:download!).with(remote_encrypted_archive, downloaded_encrypted_archive) do
+            FileUtils.cp(encrypted_archive, downloaded_encrypted_archive)
+          end
+
+          expect(Net::SFTP).to receive(:start).with(sftp_host, sftp_user, {timeout: 30}).and_return(sftp_session_double)
+          subject
+        end
+
+        it 'downloads the archive via SFTP' do; end
+
+        it 'unpacks a sql file' do
+          expect(archive_contents.size).to eq(3)
+          expect(archive_contents.last).to match(/data_export.*_db.sql/)
+          backup_contents = File.read(File.join(unpack_dir, archive_contents.last)).split("\n")
+          expect(backup_contents[0]).to match(/\A-- MySQL dump/)
+        end
+      end
+
+      context 'when no backups are found' do
+        before do
+          export(backup_key, encrypted_archive)
+
+          sftp_dir_double, sftp_session_double = double, double
+          expect(sftp_dir_double).to receive(:glob)
+          expect(sftp_session_double).to receive(:dir).and_return(sftp_dir_double)
+          expect(Net::SFTP).to receive(:start).with(sftp_host, sftp_user, {timeout: 30}).and_return(sftp_session_double)
+        end
+
+        it { expect { subject }.to raise_error SystemExit, /No backups found/ }
+      end
+    end
+  end
+
+  describe '#status_task' do
+    let(:options) { [] }
+
+    subject do
+      DataExporter::CLI.start(['status', *options, '--quiet', '--config-file', config_file.path])
+    end
+
+    let(:encrypted_archive_base_name) { 'data_export_2013-04-20-00-00_db.sql.gz.enc' }
+    let(:remote_encrypted_archive) { File.join(backup_dir, encrypted_archive_base_name) }
+
+    context 'with --redis-key-prefix' do
+      let(:options) { ['--redis-key-prefix', 'redis_key']}
+      before :each do
+        stub_s3_find_last(remote_encrypted_archive)
+        expect_any_instance_of(Redis).to receive(:set).twice
+        subject
+      end
+
+      it 'updates redis counters' do; end
+    end
+  end
+end