glacier_on_rails 0.9

data/app/models/application_database/postgres_adapter.rb

@@ -0,0 +1,52 @@
+class ApplicationDatabase::PostgresAdapter < ApplicationDatabase::BaseAdapter
+  class PgDumpCmdMissing < StandardError; end
+  class PgRestoreCmdMissing < StandardError; end
+  class PgRestoreFileMissing < StandardError; end
+
+  RestoreExclusions = %w{ application_data_backups
+                          glacier_archives
+                          SCHEMA }
+
+  RestoreList = GlacierArchive::BackupFileDir.join('restore.list')
+
+  def contents
+    `#{pg_dump} -w -Fc -U #{db_config['username']} #{db_config['database']}`
+  end
+
+  def restore(file)
+    raise PgRestoreFileMissing unless File.exists? file
+    restore_from_list(file)
+  end
+
+private
+  def generate_object_restoral_list(file)
+    # generate the raw list of all objects
+    system("#{pg_restore} --clean -n public -l #{file} > #{RestoreList}")
+    # remove objects from the list that are enumerated in RestoreExclusions
+    retained_lines = File.readlines(RestoreList).select{|line| !line.match(/#{RestoreExclusions.join("|")}/)}
+    File.open RestoreList, 'w' do |file|
+      file.write(retained_lines.join)
+    end
+  end
+
+  def restore_from_list(file)
+    generate_object_restoral_list(file)
+    result = `#{pg_restore} --clean -n public -U #{db_config['username']} --dbname=#{db_config['database']} -L #{RestoreList} #{file} 2>&1`
+    `rm #{RestoreList} && rm #{file}` if $?.exitstatus.zero?
+    AwsLog.error result unless $?.exitstatus.zero?
+    $?.exitstatus.zero?
+  end
+
+  def pg_dump
+    dump_cmd = `which pg_dump`.strip
+    raise PgDumpCmdMissing if dump_cmd.blank?
+    dump_cmd
+  end
+
+  def pg_restore
+    restore_cmd = `which pg_restore`.strip
+    raise PgRestoreCmdMissing if restore_cmd.blank?
+    restore_cmd
+  end
+
+end

data/app/models/application_database.rb

@@ -0,0 +1,29 @@
+class ApplicationDatabase
+  class MissingConfigurationKeys < StandardError
+    def initialize(missing_keys)
+      msg = "#{missing_keys.join(' and ')} must be specified in config/database.yml"
+      super(msg)
+    end
+  end
+
+  attr_accessor :adapter
+  RequiredKeys = %w{username database adapter}
+
+  def initialize
+    @adapter = case db_config["adapter"]
+               when "postgresql"
+                 PostgresAdapter.new(db_config)
+               when "mysql"
+                 MysqlAdapter.new(db_config)
+               end
+  end
+
+  def db_config
+    config = ActiveRecord::Base.configurations[Rails.env]
+    missing_keys = RequiredKeys - config.keys
+    raise MissingConfigurationKeys.new(missing_keys) unless missing_keys.empty?
+    config
+  end
+
+  delegate :contents, :restore, :to => :adapter
+end

data/app/models/application_file.rb

@@ -0,0 +1,25 @@
+require 'glacier_on_rails/config'
+class ApplicationFile
+  attr_accessor :file
+  def initialize(file)
+    # file may be a Pathname instance or a string
+    @file = file.to_s
+  end
+
+  def contents
+    ActiveSupport::Gzip.compress(File.read(file))
+  end
+
+  def filename
+    File.basename(file)
+  end
+
+  def self.list
+    files.map{|f| new(f) }.map(&:filename)
+  end
+
+  def self.files
+    Dir.glob(GlacierOnRails::Config.attached_files_directory.join('*'))
+  end
+
+end

data/app/models/aws_backend/sns_subscription.rb

@@ -0,0 +1,54 @@
+class AwsBackend
+  class SnsSubscription
+    attr_accessor :resp
+    # format is:   "arn:aws:sns:region:account-id:topicname"
+    # see http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arn-syntax-glacier
+    Topic_ARN = "arn:aws:sns:#{AwsBackend::Config.aws_region}:#{::AWS_ACCOUNT_ID}:retrieve_archive"
+    #Resource_ARN = ""
+
+    def initialize
+      client = Aws::SNS::Client.new(:region => AwsBackend::Config.aws_region, :credentials => credentials)
+      sns = Aws::SNS::Resource.new(:region => AwsBackend::Config.aws_region)
+
+      topic = sns.create_topic(name: 'retrieve_archive')
+      topic.set_attributes({
+        attribute_name: "Policy",
+        attribute_value: policy
+      })
+
+      @resp = client.subscribe({
+        topic_arn: Topic_ARN,
+        protocol: "https", # required
+        endpoint: GlacierOnRails::Engine.routes.url_helpers.aws_subscription_notify_url
+      })
+    rescue Exception => e
+      puts "AWS Error: #{e.message}"
+    end
+
+    private
+    def credentials
+      Aws::SharedCredentials.new(:profile_name => AwsBackend::Config.profile_name)
+    end
+
+    def policy
+      # see http://docs.aws.amazon.com/sdk-for-ruby/v2/developer-guide/sns-example-enable-resource.html
+      "{
+        'Version':'2014-05-06',
+        'Id':'__default_policy_ID',
+        'Statement':[{
+          'Sid':'__default_statement_ID',
+          'Effect':'Allow',
+          'Principal':{
+            'AWS':'*'
+          },
+          'Action':['SNS:Publish'],
+          'Resource':'#{Topic_ARN}',
+          'Condition':{
+            'ArnEquals':{
+              'AWS:SourceArn': #{Resource_ARN}}
+           }
+        }]
+      }"
+    end
+  end
+end

data/app/models/aws_backend.rb

@@ -0,0 +1,113 @@
+require 'glacier_on_rails/config'
+
+class AwsBackend
+  class ArchiveRetrievalNotReady < StandardError; end
+  class Config < GlacierOnRails::Config; end
+  include Singleton
+
+  attr_accessor :client, :error_message
+
+  def initialize
+    @client = Aws::Glacier::Client.new(:region => Config.aws_region, :credentials => AwsBackend.credentials)
+    unless vault_list.map(&:vault_name).include?(::SITE_NAME)
+      @client.create_vault({:account_id => "-", :vault_name => ::SITE_NAME})
+    end
+  end
+
+  def self.credentials
+    Aws::SharedCredentials.new(:profile_name => Config.profile_name)
+  end
+
+  def vault_list
+    vault_list_info = @client.list_vaults({:account_id => "-"})
+    vault_list_info.vault_list
+  end
+
+  def delete_archive(archive)
+    begin
+      response = client.delete_archive({
+        account_id: "-",
+        archive_id: archive.archive_id,
+        vault_name: ::SITE_NAME
+      })
+      AwsLog.info "Delete archive response: #{response}"
+      response
+    rescue Aws::Glacier::Errors::ServiceError => e
+      AwsLog.error "Failed to delete archive with: #{e.class}: #{e.message}"
+    end
+  end
+
+  def create_archive(archive_contents)
+    description = "backup of postgres database"
+    begin
+      resp = client.upload_archive({
+        account_id: "-",
+        archive_description: description,
+        body: archive_contents,
+        checksum: checksum(archive_contents),
+        vault_name: ::SITE_NAME
+      })
+      AwsLog.info "Create archive response: #{resp}"
+      resp
+    rescue Aws::Glacier::Errors::ServiceError => e
+      self.error_message = "Failed to create archive with: #{e.class}: #{e.message}"
+      AwsLog.error error_message
+      false
+    end
+  end
+
+  # archive is a GlacierArchive instance from the database
+  # response looks like this:
+  # <struct Aws::Glacier::Types::InitiateJobOutput location="/918359762546/vaults/demo/jobs/krCLWk6m7NJWppiy2SxdhP60f98PdrdaZBfhdDTZufrAkoh-ikrvb_NA0Q1vg2WcAhzZLL92kiwjOijUEDh0U7X09YQK", job_id="krCLWk6m7NJWppiy2SxdhP60f98PdrdaZBfhdDTZufrAkoh-ikrvb_NA0Q1vg2WcAhzZLL92kiwjOijUEDh0U7X09YQK">
+  def retrieve_archive(archive)
+    response = client.initiate_job({ account_id: "-", # required
+                          vault_name: ::SITE_NAME, # required
+                          job_parameters: {
+                            type: "archive-retrieval", # valid types are "archive-retrieval" and "inventory-retrieval"
+                            archive_id: archive.archive_id,
+                            description: "put anything here",
+                            sns_topic: SnsSubscription::Topic_ARN,
+                            tier: "Standard"# it's the default, but put it here to be explicit
+                          }
+                        })
+    AwsLog.info "Retrieve archive response: #{response}"
+    response
+  rescue Aws::Glacier::Errors::ServiceError => e
+    self.error_message = "Failed to initiate archive retrieval with: #{e.class}: #{e.message}"
+    AwsLog.error error_message
+    false
+  end
+
+  # archive is a GlacierArchive instance from the database
+  def get_job_output(archive)
+    raise ArchiveRetrievalNotReady unless  archive.retrieval_status == 'ready'
+    params = {
+      :response_target => filepath(archive),
+      :account_id => '-',
+      :vault_name => ::SITE_NAME,
+      :job_id => archive.archive_retrieval_job_id
+    }
+    AwsLog.info("AWS Backend get job output request with: #{params}")
+    resp = client.get_job_output(params)
+    AwsLog.info("AWS Backend response to get job output request: #{resp}")
+    resp
+  rescue ArchiveRetrievalNotReady
+    AwsLog.error "Get job output failed, archive status is not 'ready'"
+    false
+  end
+
+  private
+
+  def checksum(contents)
+    tree_hash = Aws::TreeHash.new
+    tree_hash.update(contents)
+    tree_hash.digest
+  end
+
+  def filepath(archive)
+    path = archive.backup_file
+    FileUtils.makedirs(File.dirname(path))
+    path
+  end
+
+end

data/app/models/aws_log.rb

@@ -0,0 +1,7 @@
+class AwsLog
+  LogFile = File.join(Rails.root, 'log', 'aws.log')
+  class << self
+    cattr_accessor :logger
+    delegate :debug, :info, :warn, :error, :fatal, :to => :logger
+  end
+end

data/app/models/glacier_archive.rb

@@ -0,0 +1,115 @@
+# GlacierArchive lifecycle:
+# ACTION                    | IMPLEMENTED BY                | PARAMETER                | STATUS INDICATION             | STATUS
+# create                    | AwsBackend#create_archive     |                          | default attributes            | available
+# initiate retrieval job    | AwsBackend#retrieve_archive   | archive_id               | archive_retrieval_job_id      | pending
+# SNS notification received | AwsSnsSubscriptionsController | archive_retrieval_job_id | notification attribute        | ready
+# fetch archive             | AwsArchiveController#fetch    | archive_retrieval_job_id | retrieved file exists locally | local
+#                           |                               |                          | original file exists locally  | exists
+class GlacierArchive < ActiveRecord::Base
+  class RestoreFail < StandardError; end
+
+  default_scope ->{ order("created_at asc") }
+
+  BackupFileDir = Rails.root.join('tmp','aws')
+
+  before_create do |archive|
+    # create the archive at AWS Glacier
+    # and save the metadata in the GlacierArchive instance
+    # archive_contents provided by subclass (GlacierDbArchive or GlacierFileArchive)
+    if archive_contents && (resp = aws.create_archive(archive_contents))
+      archive.attributes = resp.to_h
+    else
+      archive.errors.add(:base, aws.error_message)
+      throw :abort
+    end
+  end
+
+  after_destroy do |archive|
+    archive.remove_local_backup_copy
+  end
+
+  before_destroy do |archive|
+    archive.destroy_archive
+  end
+
+  def destroy_archive
+    begin
+      response = aws.delete_archive(self)
+      AwsLog.info(response)
+      true
+    rescue Aws::Glacier::Errors::ServiceError => e
+      AwsLog.error("Delete archived failed with :#{e.class}: #{e.message}")
+      false
+    end
+  end
+
+  def initiate_retrieve_job
+    if resp = aws.retrieve_archive(self)
+      update_attribute(:archive_retrieval_job_id, resp[:job_id])
+    else
+      errors.add(:base, aws.error_message)
+    end
+  end
+
+  def fetch_archive
+    begin
+      response = aws.get_job_output(self)
+      AwsLog.info(response.to_h) if response
+      true
+    rescue Aws::Glacier::Errors::ServiceError => e
+      self.errors.add(:base, e.message)
+      AwsLog.error("Fetch archive failed with: #{e.class}: #{e.message}")
+      false
+    ensure
+      reset_retrieval_status
+    end
+  end
+
+  def retrieval_status
+    exists_status || local_status || ready_status || pending_status || 'available'
+  end
+
+  def backup_file
+    BackupFileDir.join(filename)
+  end
+
+  protected
+
+  def remove_local_backup_copy
+    FileUtils.rm(backup_file) if local_status?
+  end
+
+  private
+
+  def aws
+    AwsBackend.instance
+  end
+
+  def reset_retrieval_status
+    # because error messages will be reset when update_attributes is used
+    update_columns(:archive_retrieval_job_id => nil, :notification => nil)
+  end
+
+  def exists_status
+    'exists' if exists_status?
+  end
+
+  def local_status?
+    File.exists? backup_file
+  end
+
+  # archive_retrieval job output has been retrieved
+  def local_status
+    'local' if local_status?
+  end
+
+  # ready to retrieve archive_retrieve job output
+  def ready_status
+    'ready' if notification
+  end
+
+  # archive_retrieve job has been initiated, but notification not yet received
+  def pending_status
+    'pending' if archive_retrieval_job_id
+  end
+end

data/app/models/glacier_db_archive.rb

@@ -0,0 +1,27 @@
+class GlacierDbArchive < GlacierArchive
+  belongs_to :application_data_backup
+
+  before_create do |db_archive|
+    db_archive.filename = GlacierDbArchive.filename_from_time
+  end
+
+  def self.filename_from_time
+    Time.now.strftime("%Y_%m_%d_%H_%M_%S_%L.sql").to_s
+  end
+
+  def archive_contents
+    ApplicationDatabase.new.contents
+  rescue ApplicationDatabase::MissingConfigurationKeys
+    nil
+  end
+
+  # for file archive, true inhibits fetching archive from AWS
+  # however we always want to fetch the db archive
+  def exists_status?
+    false
+  end
+
+  def restore
+    raise RestoreFail unless ApplicationDatabase.new.restore(backup_file)
+  end
+end

data/app/models/glacier_file_archive.rb

@@ -0,0 +1,45 @@
+require 'glacier_on_rails/config'
+
+class GlacierFileArchive < GlacierArchive
+  has_and_belongs_to_many :application_data_backups, join_table: 'application_data_backups_glacier_file_archives'
+
+  def file=(file)
+    self.filename = File.basename(file)
+  end
+
+  def file
+    GlacierOnRails::Config.attached_files_directory.join(filename)
+  end
+
+  def initiate_retrieve_job
+    # don't retrieve the file if it's in the filesystem already
+    # based on the assumption that files are immutable
+    super unless File.exists?(file)
+  end
+
+  # the bang method creates the instances if they didn't already exist
+  def self.all!
+    in_filesystem.collect do |file|
+      find_or_create_by(:filename => file)
+    end
+  end
+
+  def archive_contents
+    ApplicationFile.new(file).contents
+  end
+
+  def exists_status?
+    File.exists? file
+  end
+
+  def restore
+    FileUtils.mv(backup_file, file)
+  rescue Errno::ENOENT # usually b/c backup_file not found
+    raise RestoreFail
+  end
+
+  private
+  def self.in_filesystem
+    ApplicationFile.list
+  end
+end

data/app/views/glacier_on_rails/aws_archive_retrieval_jobs/_application_data_backup.haml

@@ -0,0 +1,4 @@
+%tr.application_data_backup{:outro => :fade}
+  %td.date{:style => 'width:120px;'}= application_data_backup.created_at.to_date.to_s(:default)
+  = render :partial => "glacier_on_rails/aws_archive_retrieval_jobs/#{application_data_backup.retrieval_status}", :locals => {:application_data_backup => application_data_backup, :fetch_fail => local_assigns[:fetch_fail] || false}
+  %td.action_link#delete_archive{:onclick => "delete_archive(#{application_data_backup.id})", :style => 'padding-left: 20px;'} delete

data/app/views/glacier_on_rails/aws_archive_retrieval_jobs/_available.haml

@@ -0,0 +1,3 @@
+%td#initiate_retrieval.action_link{:onclick=>"initiate_retrieval(#{application_data_backup.id})"} initiate retrieval
+- if local_assigns[:fetch_fail]
+  %td#fetch_fail_message{:style => 'padding-left:25px;'} Fetch archive failed. The job probably expired. Re-initiate archive retrieval.

data/app/views/glacier_on_rails/aws_archive_retrieval_jobs/_index.haml

@@ -0,0 +1,101 @@
+:css
+  .disabled {
+    pointer-events : none;
+  }
+  #scrim {
+    background-color : black;
+    opacity : 0.2;
+    position : absolute;
+    top : 0;
+    width : 100%;
+    height : 100%;
+  }
+  #spinner {
+    position:absolute;
+    color: white;
+    top:50%;
+    left:50%;
+    transform:translate(-50%,-50%)
+  }
+%h2 AWS Glacier Archives
+%table#application_data_backups
+  - if application_data_backups.empty?
+    %p None have been created
+  - else
+    = render :partial => 'glacier_on_rails/aws_archive_retrieval_jobs/application_data_backup', :collection => application_data_backups, :as => :application_data_backup
+
+%div{:style => 'display:flex; margin-top:12px;'}
+  %div.action_link#backup_now{:onclick => 'backup_now()'} backup now
+
+:coffeescript
+  @backup_now = ->
+    url = "#{glacier_on_rails.aws_create_archive_path}"
+    $.ajax
+      beforeSend : show_spinner
+      method : 'post'
+      url : url
+      success : add
+      complete : hide_spinner
+  show_spinner = (jqxhr, settings)->
+    $('a, .action_link').addClass('disabled')
+    $('body').append("<div id='scrim'></div><i id='spinner' class='fa fa-pulse fa-spinner fa-3x' />")
+    true
+  hide_spinner = ->
+    $('a').removeClass('disabled')
+    $('.action_link').removeClass('disabled')
+    $('#scrim').remove()
+    $('#spinner').remove()
+  add = (data, status, jqxhr)->
+    $('#application_data_backups').append(data)
+  @initiate_retrieval = (id)->
+    data = {application_data_backup_id : id}
+    url = "#{glacier_on_rails.aws_archive_retrieval_job_create_path}"
+    $.ajax
+      beforeSend : show_spinner
+      complete : hide_spinner
+      url : url
+      method : 'post'
+      data : data
+      context : $(event.target).closest('tr')
+      success : update
+  @fetch_archive = (id)->
+    url = "#{glacier_on_rails.aws_fetch_archive_path}"
+    data = {application_data_backup_id : id}
+    context = $(event.target).closest('tr')
+    $.ajax
+      beforeSend : show_spinner
+      complete : hide_spinner
+      method : 'post'
+      url : url
+      data : data
+      context : $(event.target).closest('tr')
+      success : update
+  update = (data, status, jqxhr)->
+    @replaceWith(data)
+  @restore = (id)->
+    url = "#{glacier_on_rails.aws_restore_archive_path}"
+    $.ajax
+      beforeSend : show_spinner
+      complete : hide_spinner
+      method : 'post'
+      url : url
+      data : {application_data_backup_id : id}
+      context : $(event.target).closest('tr')
+      success : restore_success #jquery handles js for error response
+  restore_success = (data, status, jqxhr)->
+    @replaceWith(data)
+    backup_date = $(data).find('.date').text()
+    flash.confirm('Database restored with the '+backup_date+' backup')
+  @delete_archive = (id)->
+    url = "#{glacier_on_rails.aws_destroy_archive_path}"
+    context = $(event.target).closest('tr')
+    $.ajax
+      beforeSend : show_spinner
+      complete : hide_spinner
+      method : 'delete'
+      url : url
+      data : {application_data_backup_id : id}
+      context : context
+      success : remove_archive
+  remove_archive = ->
+    @remove()

data/app/views/glacier_on_rails/aws_archive_retrieval_jobs/_local.haml

@@ -0,0 +1 @@
+%td.restore_database.action_link{:onclick=>"restore(#{application_data_backup.id})"} restore

data/app/views/glacier_on_rails/aws_archive_retrieval_jobs/_pending.haml

@@ -0,0 +1 @@
+%td.retrieval_pending retrieval pending

data/app/views/glacier_on_rails/aws_archive_retrieval_jobs/_ready.haml

@@ -0,0 +1,2 @@
+%td retrieval ready
+%td#fetch_archive.action_link{:style=>'padding-left:25px;',:onclick => "fetch_archive(#{application_data_backup.id})"} fetch archive

data/config/initializers/aws_log.rb

@@ -0,0 +1 @@
+AwsLog.logger = Logger.new(AwsLog::LogFile)

data/config/initializers/glacier_on_rails.rb

@@ -0,0 +1,3 @@
+GlacierOnRails::Engine.config.paths["db/migrate"].expanded.each do |expanded_path|
+  Rails.application.config.paths["db/migrate"] << expanded_path
+end

data/config/initializers/time_formats.rb

@@ -0,0 +1,3 @@
+Time::DATE_FORMATS[:short_date_with_year] = "%b %d %Y"
+Time::DATE_FORMATS[:short_time] = "%I:%M %p"
+

data/config/routes.rb

@@ -0,0 +1,9 @@
+#Rails.application.routes.draw do
+GlacierOnRails::Engine.routes.draw do
+  post :aws_subscription_notify, :to => 'aws_sns_subscriptions#create'
+  post :aws_archive_retrieval_job_create, :to => 'aws_archive_retrieval_jobs#create'
+  post :aws_fetch_archive, :to => 'application_data_backups#fetch'
+  delete :aws_destroy_archive, :to => 'application_data_backups#destroy'
+  post :aws_restore_archive, :to => 'application_data_backups#restore'
+  post :aws_create_archive, :to => 'application_data_backups#create'
+end

data/db/migrate/20170503133854_create_glacier_archives_table.rb

@@ -0,0 +1,11 @@
+class CreateGlacierArchivesTable < ActiveRecord::Migration[5.0]
+  def change
+    create_table :glacier_archives do |t|
+      t.text :description
+      t.text :archive_id
+      t.text :checksum
+      t.text :location
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20170507161533_add_notification_column_to_glacier_archives.rb

@@ -0,0 +1,5 @@
+class AddNotificationColumnToGlacierArchives < ActiveRecord::Migration[5.0]
+  def change
+    add_column :glacier_archives, :notification, :json
+  end
+end

data/db/migrate/20170509195716_add_archive_retrieval_job_id_to_glacier_archive.rb

@@ -0,0 +1,5 @@
+class AddArchiveRetrievalJobIdToGlacierArchive < ActiveRecord::Migration[5.0]
+  def change
+    add_column :glacier_archives, :archive_retrieval_job_id, :string
+  end
+end

data/db/migrate/20170602135721_create_application_data_backups.rb

@@ -0,0 +1,7 @@
+class CreateApplicationDataBackups < ActiveRecord::Migration[5.0]
+  def change
+    create_table :application_data_backups do |t|
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20170602143524_add_application_data_backup_id_to_glacier_archives.rb

@@ -0,0 +1,5 @@
+class AddApplicationDataBackupIdToGlacierArchives < ActiveRecord::Migration[5.0]
+  def change
+    add_column :glacier_archives, :application_data_backup_id, :integer
+  end
+end

data/db/migrate/20170602145526_create_glacier_file_archive_join_table.rb

@@ -0,0 +1,5 @@
+class CreateGlacierFileArchiveJoinTable < ActiveRecord::Migration[5.0]
+  def change
+    create_join_table :application_data_backups, :glacier_file_archives
+  end
+end

data/db/migrate/20170602150324_add_type_to_glacier_archives.rb

@@ -0,0 +1,5 @@
+class AddTypeToGlacierArchives < ActiveRecord::Migration[5.0]
+  def change
+    add_column :glacier_archives, :type, :string
+  end
+end