data-exporter 1.3.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 29fcc31fa3466058bc785fe88e2e1acadec677d7
+  data.tar.gz: 8a41c38d5c0383ce63ae6290f88406e82f32c279
+SHA512:
+  metadata.gz: 9ac800e453a9623a806dc87b87d7a8878d5ab78c59b36a7c9d84a46524c3b2567713fce70e3e05af5039bb205711c5f9ed2043f401f3e98ca3a6c7260601b892
+  data.tar.gz: d490349507e86a2f597e9ab0e58ed1fccb6d402516d3566be748b3abfd53bc583f244e73a60674b00cb94575442e3e7968f4b99979e0187ecf1982561e0e17ba

data/README.md

@@ -0,0 +1,40 @@
+DataExporter
+------------
+Use to export and import MySQL databases to S3.
+
+To export a MySQL database:
+
+1. Add gem to Gemfile
+2. Create a YAML or ERB configuration that `data-exporter` can read:
+```yaml
+export_dir: '/tmp'
+backup_dir: '/backups'
+backup_key: 'config/backup_key' # openssl encryption key file
+mysql:
+  adapter: 'mysql2'
+  host: 'localhost'
+  database: 'centurion_development'
+  username: 'root'
+  password: ''
+s3:
+  access_key_id: 'ACCESS_KEY_ID'
+  secret_access_key: 'SECRET_ACESS_KEY'
+  bucket_name: 'socialcast_backups'
+  prefix: 'centurion_development'
+```
+
+3. Execute:
+```shell
+bundle exec data-exporter export -f etc/data_exporter.yml --csv
+```
+
+
+To import a MySQL database as a directory of CSV files:
+```shell
+bundle exec data-exporter unpack -f etc/data_exporter.yml --csv
+```
+
+To import a MySQL database backup via --sftp:
+```shell
+bundle exec data-exporter unpack -f etc/data_exporter.yml --csv --sftp
+```

data/Rakefile

@@ -0,0 +1,15 @@
+#!/usr/bin/env rake
+
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/data-exporter

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+$: << File.expand_path('../../lib/', __FILE__)
+require 'data_exporter'
+DataExporter::CLI.start

data/lib/data_exporter.rb

@@ -0,0 +1,21 @@
+module DataExporter
+  require 'data_exporter/configuration'
+  require 'data_exporter/archive'
+  require 'data_exporter/actions'
+  require 'data_exporter/cli'
+
+  class << self
+    def configuration
+      @configuration ||= DataExporter::Configuration.new
+    end
+    alias :config :configuration
+
+    def configure(&block)
+      yield configuration
+    end
+
+    def database_connection
+      ActiveRecord::Base.establish_connection(configuration.database)
+    end
+  end
+end

data/lib/data_exporter/actions.rb

@@ -0,0 +1,321 @@
+require 'active_record'
+require 'csv'
+require 'open3'
+require 'net/sftp'
+require 'aws/s3'
+require 'ostruct'
+
+module DataExporter
+  IGNORED_TABLES = %w(schema_migrations checksums)
+
+  module Actions
+    def decrypt(in_file_name = '-')
+      ['openssl', 'enc', '-d', '-aes-256-cbc', '-salt', '-pass', "file:#{config.backup_key}", '-in', in_file_name]
+    end
+
+    def expand(file_name = '-')
+      ['gunzip', file_name]
+    end
+
+    def unarchive(file_name = '-')
+      ['tar', '-xf', file_name]
+    end
+
+    def compress(file_name = '-')
+      ['gzip', '--to-stdout', file_name]
+    end
+
+    def encrypt(backup_key)
+      ['openssl', 'enc', '-aes-256-cbc', '-salt', '-pass', "file:#{backup_key}"]
+    end
+
+    def unpack(encrypted_file, unpack_dir)
+      case encrypted_file
+      when /\.tar\.gz\.enc\z/
+        unpack_encrypted_archive(encrypted_file, unpack_dir)
+      when /\.sql\.gz\.enc\z/
+        unpack_encrypted_file(encrypted_file, unpack_dir)
+      end
+    end
+
+    def export(backup_key, encrypted_file, archive_dir = nil)
+      case encrypted_file
+      when /\.tar\.gz\.enc\z/
+        export_encrypted_archive(backup_key, encrypted_file, archive_dir)
+      when /\.sql\.gz\.enc\z/
+        export_encrypted_file(backup_key, encrypted_file)
+      end
+    end
+
+    def config
+      DataExporter.config
+    end
+
+    def find_last_backup(prefix, suffix, backup_dir)
+      if config.sftp_enabled?
+        find_last_sftp_backup(prefix, suffix, backup_dir)
+      else
+        find_last_s3_backup(prefix, suffix, backup_dir)
+      end
+    end
+
+    def find_last_sftp_backup(prefix, suffix, backup_dir = '/')
+      backup_entry = nil
+      sftp.dir.glob(backup_dir, prefix + '*' + suffix) do |entry|
+        backup_entry ||= entry
+        if entry.attributes.mtime > backup_entry.attributes.mtime
+          backup_entry = entry
+        end
+      end
+      OpenStruct.new(:name => File.join(backup_dir, backup_entry.name), :mtime => backup_entry.attributes.mtime, :size => backup_entry.attributes.size) if backup_entry
+    rescue => e
+      log "#{e.to_s}"
+      nil
+    end
+
+    def glob_escape(glob)
+      glob.gsub('.', '\\.').gsub('*', '.*')
+    end
+
+    def find_last_s3_backup(prefix, suffix, backup_dir = nil)
+      s3_backup = nil
+      s3_bucket = config.s3[:bucket]
+      s3_prefix = backup_dir ? File.join(backup_dir, prefix) : prefix
+      s3.buckets[s3_bucket].objects.with_prefix(s3_prefix).each do |s3_object|
+        next unless s3_object.key =~ /#{glob_escape(suffix)}\Z/
+        s3_backup = s3_object
+      end
+      OpenStruct.new(:name => s3_backup.key, :mtime => s3_backup.last_modified.to_i, :size => s3_backup.content_length, :io => s3_backup) if s3_backup
+    rescue => e
+      log "#{e.to_s}"
+      nil
+    end
+
+    private
+
+    SFTP_MAX_TIMEOUT = 30
+
+    def pipeline_with_log(*args)
+      opts = Hash === args.last ? args.pop.dup : {}
+      err_r, err_w = IO.pipe
+      threads = []
+      process = 0
+      Open3.pipeline_start(*args, opts.merge(:err => err_w)) do |ts|
+        ts.each { |t| wait_unless_done(t.value) }
+        err_w.close
+        log(err_r.read)
+        ts.each { |x| log("executed '#{args[process].join( ' ')}' - #{x.value}"); process += 1 }
+        threads += ts
+      end
+      threads.map(&:value)
+    end
+
+    def wait_unless_done(process)
+      Process.wait(process.pid) if process && process.try(:pid)
+    rescue
+    end
+
+    def unpack_encrypted_file(encrypted_file, unpack_dir)
+      FileUtils.mkdir_p(unpack_dir)
+      out_file_name = File.join(unpack_dir, File.basename(encrypted_file).chomp('.gz.enc'))
+      status = pipeline_with_log(decrypt(encrypted_file), expand, :out => out_file_name)
+      raise SystemExit, "Problem unpacking #{encrypted_file}" unless status.all? { |x| x.success? }
+    end
+
+    def unpack_encrypted_archive(encrypted_archive, unpack_dir)
+      tmp_unpack_dir = Dir.mktmpdir('data_exporter')
+
+      status = Dir.chdir(tmp_unpack_dir) do
+        pipeline_with_log(decrypt(encrypted_archive), expand, unarchive)
+      end
+      raise SystemExit, "Problem unpacking #{encrypted_archive}" unless status.all? { |x| x.success? }
+
+      archive_unpack_dir = File.join(tmp_unpack_dir, config.archive_base_directory, '/')
+      FileUtils.rmtree(unpack_dir)
+      FileUtils.move(archive_unpack_dir, unpack_dir)
+    end
+
+    def export_encrypted_archive(backup_key, encrypted_archive, archive_dir = nil)
+      local_archive_name = File.join(config.export_dir, File.basename(encrypted_archive).chomp('.gz.enc'))
+      archive_dir ? archive_dir_csv_export(local_archive_name, archive_dir) : mysql_csv_export(local_archive_name)
+      pipeline_with_log(compress(local_archive_name), encrypt(backup_key), :out => encrypted_archive)
+      log "removing #{local_archive_name}"
+      FileUtils.rm(local_archive_name)
+    end
+
+    def export_encrypted_file(backup_key, encrypted_file)
+      pipeline_with_log(mysqldump_export, compress, encrypt(backup_key), :out => encrypted_file)
+    end
+
+    def download(remote_file_info, download_dir)
+      if config.sftp_enabled?
+        download_via_sftp(remote_file_info, download_dir)
+      else
+        download_via_s3(remote_file_info, download_dir)
+      end
+    end
+
+    def download_via_sftp(remote_file_info, local_dir)
+      local_file = nil
+      local_file = File.join(local_dir, File.basename(remote_file_info.name))
+      sftp.download!(remote_file_info.name, local_file) do |event, downloader, *args|
+        case event
+        when :open then
+          # args[0] : file metadata
+          log "downloading #{args[0].remote} -> #{args[0].local} (#{args[0].size} bytes)"
+        when :close then
+          # args[0] : file metadata
+          log "finished #{args[0].remote}"
+        end
+      end
+      local_file
+    end
+
+    def download_via_s3(remote_file_info, local_dir)
+      local_file = File.join(local_dir, File.basename(remote_file_info.name))
+      log "downloading #{remote_file_info.name} to #{local_file}"
+      File.open(local_file, 'w') do |file|
+        remote_file_info.io.read do |chunk|
+          file.write(chunk)
+        end
+      end
+      local_file
+    end
+
+    def upload(local_file, remote_dir)
+      if config.sftp_enabled?
+        upload_via_sftp(local_file, remote_dir)
+      else
+        upload_via_s3(local_file, remote_dir)
+      end
+    end
+
+    def upload_via_sftp(local_file, backup_dir = nil)
+      remote_file = backup_dir ? File.join(backup_dir, File.basename(local_file)) : File.basename(local_file)
+      sftp.mkdir(backup_dir)
+      sftp.upload!(local_file, remote_file) do |event, uploader, *args|
+        case event
+        when :open then
+          # args[0] : file metadata
+          log "uploading #{args[0].local} -> #{args[0].remote} (#{args[0].size} bytes)"
+        when :close then
+          # args[0] : file metadata
+          log "finished #{args[0].remote}"
+        end
+      end
+    end
+
+    def upload_via_s3(local_file, backup_dir = nil)
+      s3_bucket = config.s3[:bucket]
+      s3_file = backup_dir ? File.join(backup_dir, File.basename(local_file)) : File.basename(local_file)
+      log "uploading #{local_file} to s3://#{s3_bucket}/#{s3_file}"
+      s3.buckets[s3_bucket].objects[s3_file].write(open(local_file))
+    end
+
+    def archive_dir_csv_export(archive_name, archive_dir)
+      log "creating #{archive_name}"
+      DataExporter::Archive.open(archive_name) do |archive|
+        Dir.glob(File.join(archive_dir, '*')).each do |file_name|
+          archive_file_name = File.join(config.archive_base_directory, File::basename(file_name))
+          log "appending #{archive_file_name}"
+          archive.append(archive_file_name, file_name)
+        end
+      end
+    end
+
+    def mysql_csv_export(archive_name)
+      log "creating #{archive_name}"
+      DataExporter::Archive.open(archive_name) do |archive|
+        mysql_to_csv do |file_name|
+          archive_file_name = File.join(config.archive_base_directory, File::basename(file_name))
+          log "appending #{archive_file_name}"
+          archive.append(archive_file_name, file_name)
+          FileUtils.rm(file_name)
+        end
+      end
+    end
+
+    # NOTE database lock is necessary for consistency
+    def mysql_to_csv(&block)
+      begin
+        connection_pool = DataExporter::database_connection
+        connection_pool.connection.tables.each do |table|
+          next if IGNORED_TABLES.include?(table)
+          fields = connection_pool.connection.columns(table).map(&:name)
+          total =  0
+          connection_pool.connection.select_rows("SELECT COUNT(*) AS count FROM #{table};").each { |result| total = result[0] }
+          selected_fields = fields - config.pii_fields(table)
+          page = 1
+          block_size = 500_000
+          total_number_of_pages = (total.to_f / block_size.to_f).ceil
+          last_id = 0
+          quoted_selected_fields = selected_fields.map { |field| connection_pool.connection.quote_column_name(field) }
+          while page <= total_number_of_pages do
+            csv_file = "#{File.join(config.export_dir, "#{table}_#{page}")}.csv"
+            CSV.open(csv_file, "w") do |csv|
+              csv << selected_fields
+              results = connection_pool.connection.select_rows("SELECT #{quoted_selected_fields.join(',')} FROM #{table} #{"WHERE id > #{last_id} ORDER BY id ASC LIMIT #{block_size}" if fields.include?('id')};")
+              if results
+                results.each do |table_values|
+                  begin
+                    csv << table_values
+                  rescue => e
+                    STDERR.puts "skipping #{table}.id = #{table_values[0]} - #{e.to_s}"
+                  rescue => g
+                    STDERR.puts "#{g.to_s}"
+                  end
+                end
+                last_id = results.last[0] if fields.include?('id')
+              end
+            end
+            yield csv_file
+            page += 1
+          end
+        end
+      ensure
+        connection_pool.disconnect!
+      end
+    end
+
+    def mysqldump_export
+      [
+        config.mysqldump_path,
+        *config.mysqldump_options,
+        config.database[:host] ? "--host=#{config.database[:host]}" : nil,
+        config.database[:port] ? "--port=#{config.database[:port]}" : nil,
+        "--user=#{config.database[:username]}",
+        config.database[:password] ? "--password=#{config.database[:password]}" : nil,
+        config.database[:database]
+      ].compact
+    end
+
+    def log(*a); end
+
+    def https_proxy
+      ENV['https_proxy'] || ENV['HTTPS_PROXY']
+    end
+
+    def s3_options
+      @s3_options ||=
+      {
+        :access_key_id => config.s3[:access_key_id],
+        :secret_access_key => config.s3[:secret_access_key],
+        :proxy_uri => https_proxy,
+        :use_ssl => true
+      }
+    end
+
+    def s3
+      @s3 ||= AWS::S3.new(s3_options)
+    end
+
+    def sftp
+      @sftp ||= Net::SFTP.start(config.sftp[:host], config.sftp[:user], timeout: SFTP_MAX_TIMEOUT)
+    end
+
+    def redis
+      require 'redis'
+      @redis ||= Redis.new(:host => config.redis[:host], :port => config.redis[:port])
+    end
+  end
+end

data/lib/data_exporter/archive.rb

@@ -0,0 +1,25 @@
+require 'archive/tar/minitar'
+
+module DataExporter
+  class Archive
+    class << self
+      def open(archive_filename, &block)
+        File.open(archive_filename, 'w') do |file|
+          ::Archive::Tar::Minitar::Writer.open(file) do |writer|
+            yield self.new(writer)
+          end
+        end
+      end
+    end
+
+    def initialize(writer)
+      @writer = writer
+    end
+
+    def append(archive_filename, filename)
+      @writer.add_file(archive_filename, :mode => 0644, :mtime => Time.now) do |archive_file|
+        archive_file.write File.read(filename)
+      end
+    end
+  end
+end

data/lib/data_exporter/cli.rb

@@ -0,0 +1,154 @@
+require 'thor'
+require 'logger'
+
+module DataExporter
+  class CLI < Thor
+    include DataExporter::Actions
+
+    namespace :data
+    class_option 'debug', :aliases => '-d', :desc => 'Turn on debug logging', :default => false
+    class_option 'csv', :desc => 'perform csv unpack', :default => false
+    class_option 'quiet', :aliases => '-q', :desc => 'silence diagnostic information', :default => false
+    class_option 'config_file', :aliases => '-f', :desc => 'configuration file to load', :required => true
+    class_option 'preserve', :aliases => '-p', :desc => 'preserve unpack after download', :default => false
+    class_option 'mode', :desc => 'backup mode, corresponds to section data_exporter.yml configuration file'
+
+    desc 'unpack', 'unpack mysql database'
+    method_option 'date', :desc => 'unpack export for date'
+    method_option 'unpack_dir', :desc => 'directory to unpack export into'
+    def unpack_task
+      config.load(options.merge(:mysql_required => false))
+
+      remote_backup = find_last_backup(config.backup_prefix, backup_suffix, config.backup_dir)
+
+      abort no_backups_message unless remote_backup
+
+      FileUtils.mkdir_p(config.download_dir)
+      FileUtils.mkdir_p(File.dirname(config.unpack_dir))
+
+      begin
+        local_encrypted_archive = download(remote_backup, config.download_dir)
+        log "expanding #{local_encrypted_archive}"
+        unpack(local_encrypted_archive, config.unpack_dir)
+      ensure
+        unless options[:preserve]
+          log "removing #{local_encrypted_archive}"
+          FileUtils.rm(local_encrypted_archive)
+        end
+      end
+    end
+
+    desc 'export', 'export mysql database'
+    method_option 'pii_file', :desc => 'tables and columns labeled as pii'
+    method_option 'archive_dir', :desc => 'local directory to export'
+    method_option 'date', :desc => 'date of export'
+    def export_task
+      config.load(options.merge(:mysql_required => !options[:archive_dir]))
+      raise ArgumentError, '--csv required for --archive-dir' if options[:archive_dir] && !config.csv_enabled?
+
+      local_encrypted_archive_name = File.join(config.export_dir, encrypted_export_archive_name)
+      log "creating #{local_encrypted_archive_name}"
+      export(config.backup_key, local_encrypted_archive_name, options[:archive_dir])
+
+      begin
+        upload(local_encrypted_archive_name, config.backup_dir)
+      ensure
+        unless options[:preserve]
+          log "removing #{local_encrypted_archive_name}"
+          FileUtils.rm(local_encrypted_archive_name)
+        end
+      end
+    end
+
+    desc 'status', 'display status of current exported backups'
+    method_option 'date', :desc => 'unpack export for date'
+    method_option 'redis_key_prefix', :desc => 'redis_key_prefix for monitoring keys'
+    def status_task
+      config.load(options)
+
+      remote_backup = find_last_backup(config.backup_prefix, backup_suffix, config.backup_dir)
+
+      abort no_backups_message unless remote_backup
+      log("last backup %s at %s (%s)" % [remote_backup.name, Time.at(remote_backup.mtime).utc.iso8601, bytes(remote_backup.size)])
+
+      update_redis_counters(remote_backup, options[:redis_key_prefix]) if options[:redis_key_prefix]
+    end
+
+    private
+
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+
+    def log(*a)
+      if options[:debug]
+        logger.info(*a)
+      else
+        say *a unless options[:quiet]
+      end
+    end
+
+    def export_date_format
+      (options[:date] ? Date.parse(options[:date]) : Time.now).strftime("%Y-%m-%d-%H-%M")
+    end
+
+    def export_base_name
+      "#{config.backup_prefix}_#{export_date_format}_db"
+    end
+
+    def export_archive_name
+      if config.csv_enabled?
+        [export_base_name, 'csv', 'tar'].join('.')
+      else
+        [export_base_name, 'sql'].join('.')
+      end
+    end
+
+    def encrypted_export_archive_name
+      [export_archive_name, 'gz', 'enc'].join('.')
+    end
+
+    def backup_suffix
+      suffix =
+      if config.csv_enabled?
+        ['csv', 'tar', 'gz', 'enc']
+      else
+        ['sql', 'gz', 'enc']
+      end
+      suffix.unshift "#{options[:date]}*_db" if options[:date]
+      suffix.join('.')
+    end
+
+    def no_backups_message
+      if config.sftp_enabled?
+        "No backups found in #{config.backup_dir} matching #{config.backup_prefix}.*#{backup_suffix}"
+      else
+        "No backups found in #{config.s3[:bucket]}/#{config.backup_dir} matching #{config.backup_prefix}.*#{backup_suffix}"
+      end
+    end
+
+    def bytes(size)
+      {GB: 30, MB: 20, KB: 10}.each do |prefix, pow|
+        b = size >> pow
+        return "#{b} #{prefix}" if b > 0
+      end
+      return "#{size} B"
+    end
+
+    def update_redis_counters(remote_backup, redis_key_prefix)
+      counters = {}
+      if config.csv_enabled?
+        counters["#{redis_key_prefix}:mysql_csv_last_backup_timestamp"] = remote_backup.mtime
+        counters["#{redis_key_prefix}:mysql_csv_last_backup_size"] = remote_backup.size
+      else
+        counters["#{redis_key_prefix}:mysql_last_backup_timestamp"] = remote_backup.mtime
+        counters["#{redis_key_prefix}:mysql_last_backup_size"] = remote_backup.size
+      end
+
+      counters.each do |key, val|
+        log "setting redis #{key} to #{val}"
+        redis.set(key, val)
+      end
+    end
+  end
+end