cryo 0.0.1 → 0.0.2

data/.gitignore

@@ -15,3 +15,8 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+*~
+cryo.rc
+\#*\#
+.\#*
+cryo.arx

data/Makefile

@@ -0,0 +1,5 @@
+cryo.arx: 
+	git archive HEAD | bzip2 | arx tmpx -rm! - -e ./bin/bootstrap_cryo.sh FOO=bar > cryo.arx
+
+clean: 
+	rm cryo.arx

data/bin/bootstrap_cryo.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -e
+
+echo hey there

data/bin/cryo

@@ -0,0 +1,158 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/cryo"
+require 'trollop'
+
+required_parameters = ["archive_bucket",
+                       "archive_frequency",
+                       "aws_access_key",
+                       "aws_secret_key",
+                       "host",
+                       "password",
+                       "snapshot_bucket",
+                       "snapshot_frequency",
+                       "snapshot_period",
+                       "snapshot_prefix",
+                       "sns_topic",
+                       "tmp_path",
+                       "user",
+                      ]
+
+mode = ARGV.shift
+
+options = Trollop::options do 
+  version "cryo #{Cryo::VERSION} (c) 2013 Airbnb"
+  banner <<-END_OF_BANNER
+
+#{version}
+
+Welcome to Cryo, a simple backup utility
+
+All options can be passed in with either the given command line options, or passed in via
+environment with the same name, capitalized, and with CRYO_ appended. Like: CRYO_HOST instead of --host
+
+More docs and examples can be found at https://github.com/airbnb/cryo
+
+Usage: 
+      cryo [redis,mysql,postgres,list,get] [options]
+
+where [options] are:
+
+END_OF_BANNER
+
+  opt(:tmp_path,
+      "where should temp files be created",
+      :type => String,
+      :default => ENV['CRYO_TMP_PATH'])
+  
+  opt(:snapshot_frequency,
+      "how often to take backups (in mins)",
+      :type => Integer,
+      :default => ENV['CRYO_SNAPSHOT_FREQUENCY'].to_i)
+  
+  opt(:archive_frequency,
+      "maxium time in between archives (in mins)",
+      :type => Integer,
+      :default => ENV['CRYO_ARCHIVE_FREQUENCY'].to_i)
+
+  opt(:snapshot_period,
+      "time before snapshots get deleted or archived (in mins)",
+      :type => Integer, 
+      :default => ENV['CRYO_SNAPSHOT_PERIOD'].to_i)
+
+  opt(:snapshot_bucket,
+      "s3 bucket to use for snapshots",
+      :type => String,
+      :default => ENV['CRYO_SNAPSHOT_BUCKET'])
+
+  opt(:snapshot_prefix,
+      "s3 object prefix to use for snapshots",
+      :type => String,
+      :default => ENV['CRYO_SNAPSHOT_PREFIX'])
+
+  opt(:archive_bucket,
+      "s3 bucket to use for archives",
+      :type => String,
+      :default => ENV['CRYO_ARCHIVE_BUCKET'])
+
+  opt(:archive_prefix,
+      "s3 object prefix to use for archives",
+      :type => String,
+      :default => ENV['CRYO_ARCHIVE_PREFIX'])
+
+  opt(:sns_topic,
+      "sns topic",
+      :type => String,
+      :default => ENV['CRYO_SNS_TOPIC'])
+
+  opt(:aws_access_key,
+      "aws_access_key. Can be set using the AWS_ACCESS_KEY environment variable",
+      :type => String,
+      :default => ENV['CRYO_AWS_ACCESS_KEY'])
+
+  opt(:aws_secret_key,
+      "aws_secret_key. Can be set using the AWS_SECRET_KEY environment variable",
+      :type => String,
+      :default => ENV['CRYO_AWS_SECRET_KEY'])
+
+  opt(:host,
+      "remote host. Can be set using the CRYO_HOST environment variable",
+      :type => String,
+      :default => ENV['CRYO_HOST'])
+
+  opt(:user,
+      "remote user",
+      :type => String,
+      :default => ENV['CRYO_USER'])
+
+  opt(:password,
+      "remote password",
+      :type => String,
+      :default => ENV['CRYO_PASSWORD'])
+
+
+  case mode 
+  when 'mysql'
+  when 'redis'
+    opt(:path,
+        "path to redis database file",
+        :type => String,
+        :default => ENV['CRYO_PATH'])
+    required_parameters << 'path'
+  when 'postgres'
+  when 'list'
+  when 'get'
+  else
+    unless mode == '--help'
+      STDERR.puts "ERROR! bad input. first option needs to be one of [redis, postgres, mysql, list, get]"
+      STDERR.puts "Please use --help for more info"
+      exit 1 
+    end
+  end
+end
+
+options.merge!(type: mode)
+
+
+required_parameters.each do |arg|
+  Trollop::die arg.to_sym, "needs to specified on the commandline or set by the CRYO_#{arg.upcase} environment variable" \
+    if options[arg.to_sym].nil? or ! options[arg.to_sym]
+end unless mode == 'get'
+
+
+run = Cryo.new(options)
+
+case mode 
+when 'list'
+  run.list_snapshots
+when 'get'
+  snapshot = ARGV.shift
+  raise "you did not specify a snapshot!" if snapshot.nil?
+  run.get_snapshot(snapshot)
+else
+  run.backup!
+  run.archive_and_purge
+end
+
+# 
+# run.list_archives

data/bin/test

@@ -0,0 +1,100 @@
+options = Trollop::options do 
+
+  opt(:snapshot_frequency,
+      "how often to take backups (in mins)",
+      :type => Integer,
+      :default => ENV['CRYO_SNAPSHOT_FREQUENCY'].to_i || nil,
+      :required => true)
+  
+  opt(:archive_frequency,
+      "maxium time in between archives (in mins)",
+      :type => Integer,
+      :default => ENV['CRYO_ARCHIVE_FREQUENCY'].to_i || nil,
+      :required => true)
+
+  opt(:snapshot_period,
+      "time before snapshots get deleted or archived (in mins)",
+      :type => Integer, 
+      :default => ENV['CRYO_SNAPSHOT_PERIOD'].to_i || nil,
+      :required => true)
+
+  opt(:snapshot_bucket,
+      "s3 bucket to use for snapshots",
+      :type => String,
+      :default => ENV['CRYO_SNAPSHOT_BUCKET'] || nil,
+      :required => true)
+
+  opt(:snapshot_prefix,
+      "s3 object prefix to use for snapshots",
+      :type => String,
+      :default => ENV['CRYO_SNAPSHOT_PREFIX'] || nil,
+      :required => true)
+
+  opt(:archive_bucket,
+      "s3 bucket to use for archives",
+      :type => String,
+      :default => ENV['CRYO_ARCHIVE_BUCKET'] || nil,
+      :required => true)
+
+  opt(:archive_prefix,
+      "s3 object prefix to use for archives",
+      :type => String,
+      :default => ENV['CRYO_ARCHIVE_PREFIX'] || nil,
+      :required => true)
+
+  opt(:sns_topic,
+      "sns topic",
+      :type => String,
+      :default => ENV['CRYO_SNS_TOPIC'] || nil,
+      :required => true)
+
+
+
+
+
+
+  opt(:host,
+      "remote host. Can be set using the CRYO_HOST environment variable",
+      :type => String,
+      :default => ENV['CRYO_HOST'] || nil,
+      :required => true)
+
+  opt(:user,
+      "remote user",
+      :type => String,
+      :default => ENV['CRYO_USER'] || nil,
+      :required => true)
+
+  opt(:password,
+      "remote password",
+      :type => String,
+      :default => ENV['CRYO_PASSWORD'] || nil,
+      :required => true)
+
+
+  case mode 
+  when 'mysql'
+  when 'redis'
+    opt(:path,
+        "path to redis database file",
+        :type => String,
+        :default => ENV['CRYO_PATH'])
+    required_parameters << 'path'
+  when 'postgres'
+  else
+    unless mode == '--help'
+      STDERR.puts "ERROR! bad input. first option needs to be one of [redis, postgres, mysql]"
+      STDERR.puts "Please use --help for more info"
+      exit 1 
+    end
+  end
+end
+
+options.merge!(type: mode)
+
+
+
+  puts "optionss are:"
+  require 'pp'
+  pp options
+

data/bin/test2

@@ -0,0 +1,210 @@
+#!/usr/bin/env ruby
+
+require 'trollop'
+require_relative "../lib/cryo"
+
+
+@required_inputs = [
+                    'aws_access_key',
+                    'aws_seceret_key',
+                   ]
+
+
+@options={}
+
+@noun=nil
+@verb=nil
+
+@log=true
+
+def log(msg='')
+  STDERR.puts msg if @log
+end
+
+
+def print_help_and_exit
+  banner = <<-END_OF_BANNER
+Welcome to Cryo, a simple backup utility. #{Cryo::VERSION} (c) 2013 Airbnb
+
+All options can be passed in with either the given command line options, or passed in via
+environment with the same name, capitalized, and with CRYO_ appended. Like: CRYO_HOST instead of --host
+
+More docs and examples can be found at https://github.com/airbnb/cryo
+
+Usage: 
+      cryo [backup,list,get] [options]
+
+where [options] are:
+
+END_OF_BANNER
+  puts banner
+  Kernel.exit 1
+end
+
+def print_message_and_exit(message='')
+  STDERR.puts message
+  Kernel.exit 1
+end
+
+def parse_inputs
+  log "starting to parse command line inputs"
+  while next_argument = ARGV.shift
+    log "looping inside parse_inputs. next arg is #{next_argument}"
+    print_help_and_exit unless next_argument.start_with? '--'
+    formatted_key = next_argument.gsub(/^--/,'').gsub(/-/,'_').to_sym
+    if !ARGV.empty? and !ARGV.first.start_with? '--'
+      log "this option has a param"
+      param = ARGV.shift
+    else
+      log "this arg does not have a param"
+      param = true
+    end
+    @options.merge!({formatted_key => param})
+  end
+end
+
+def verify_environment_variable(variable='')
+  log "checking for environment variable #{variable}"
+  formatted_variable_name = "CRYO_#{variable.upcase}"
+  value = ENV[formatted_variable_name]
+  unless value.nil?
+    log "looks like the variable exists"
+    @options.merge!({variable.to_sym => value})
+  else
+    log "looks like the variable does not exist"
+    print_message_and_exit "\nyou need to make sure that you set the #{formatted_variable_name} variable!!!"
+  end
+end
+
+def verify_inputs
+  @required_inputs.each do |input|
+    verify_environment_variable input
+  end
+end
+
+
+def backup
+  %w{host}.each {|i| @required_inputs << i}
+  case @noun
+  when "mysql"
+  when "redis"
+  when "postgres"
+  else
+    print_help_and_exit
+  end
+end
+
+
+def list
+  case @noun
+  when "snapshots"
+  when "archives"
+  else
+    print_help_and_exit
+  end
+end
+
+
+def get
+  case @noun
+  when "archives"
+  when "snapshots"
+  else
+    print_help_and_exit
+  end
+end
+
+
+log "starting to parse arguments"
+print_help_and_exit if ARGV.size < 2
+@verb = ARGV.shift
+@noun = ARGV.shift
+log "got a noun and a verb. #{@verb} and #{@noun}"
+
+case @verb
+when "backup"
+  backup
+when "list"
+  list
+when "get"
+  get
+else
+  print_help_and_exit
+end
+
+parse_inputs
+
+
+verify_inputs
+
+log "options are #{@options.inspect}"
+
+Kernel.exit 2
+
+
+SUB_COMMANDS = %w(backup list get)
+global_opts = Trollop::options do
+  STDERR.puts "entering global options"
+  stop_on_unknown
+  version "cryo #{Cryo::VERSION} (c) 2013 Airbnb"
+
+  
+  opt(:aws_access_key,
+      "aws_access_key. Can be set using the AWS_ACCESS_KEY environment variable",
+      :type => String,
+      :default => ENV['CRYO_AWS_ACCESS_KEY'] || nil,
+      :required => true)
+  stop_on SUB_COMMANDS
+end
+
+
+
+
+
+cmd = ARGV.shift # get the subcommand
+cmd_opts = \
+case cmd
+when "backup" # parse delete options
+  Trollop::options do
+    stop_on_unknown
+    stop_on ["aws_access_key"]
+    STDERR.puts "entering backup options"
+    opt :force, "Force deletion"
+  end
+when "list"  # parse copy options
+  Trollop::options do
+    STDERR.puts "entering list options"
+    opt :double, "Copy twice for safety's sake"
+  end
+when "get"  # parse copy options
+  Trollop::options do
+    STDERR.puts "entering get options"
+    opt :double, "Copy twice for safety's sake"
+  end
+when nil
+  STDERR.puts "entering nil options"
+  puts "please call with --help if you need help"
+  Kernel.exit 1
+else
+  STDERR.puts "entering else options"
+  Trollop::die "unknown subcommand #{cmd.inspect}"
+end
+
+always_required_options = Trollop::options do 
+  STDERR.puts "entering always required options"
+  stop_on_unknown
+
+
+  opt(:aws_secret_key,
+      "aws_secret_key. Can be set using the AWS_SECRET_KEY environment variable",
+      :type => String,
+      :default => ENV['CRYO_AWS_SECRET_KEY'] || nil,
+      :required => true)
+
+end
+
+
+puts "Global options: #{global_opts.inspect}"
+puts "Subcommand: #{cmd.inspect}"
+puts "Subcommand options: #{cmd_opts.inspect}"
+puts "Remaining arguments: #{ARGV.inspect}"

data/cryo.gemspec

@@ -4,13 +4,20 @@ require File.expand_path('../lib/cryo/version', __FILE__)
 Gem::Specification.new do |gem|
   gem.authors       = ["Nathan Baxter"]
   gem.email         = ["nathan.baxter@airbnb.com"]
-  gem.summary       = %q{Tool for snapshotting data, backing it up, verifying it, and cycling it.}
+  gem.summary       = %q{Tool for snapshotting data, backing it up, verifying it, cycling it, and triggering notifications.}
   gem.homepage      = "https://github.com/airbnb/cryo"
 
   gem.files         = `git ls-files`.split($\)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.executables   = ["cryo"]
+
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "cryo"
   gem.require_paths = ["lib"]
   gem.version       = Cryo::VERSION
+
+  gem.add_runtime_dependency "colorize"
+  gem.add_runtime_dependency "trollop", "~> 2.0"
+  gem.add_runtime_dependency "aws-sdk", '~> 1.6'
+  gem.add_runtime_dependency "net-ntp", '~> 2.1.1'
+  gem.add_development_dependency "pry"
 end

data/examples/rds.sh

@@ -0,0 +1,28 @@
+#!/bin/bash -ex
+
+cd $(dirname $0) && cd ..
+
+./bin/cryo redis\
+  --host localhost \
+  --user me \
+  --password verysafe \
+  --sns-topic arn:aws:sns:us-east-1:172631448019:martin-redis-test \
+  --key somekey \
+  --aws-access-key some_aws_access_key \
+  --aws-secret-key some_secret \
+  --bucket some_buck \
+  --path /mnt/redis/foo \
+
+
+# or
+
+export CRYO_AWS_ACCESS_KEY=some_key
+export CRYO_AWS_SECRET_KEY=some_secret
+export CRYO_BUCKET=some_other_buk
+export CRYO_SNS_TOPIC=some_sns_topic
+export CRYO_HOST=some_server_somewhere
+export CRYO_USER=some_user
+export CRYO_PASSWORD=some_good_password
+export CRYO_PATH=/some/path/to/redis/db
+
+./bin/cryo mysql

data/lib/cryo.rb

@@ -1,5 +1,90 @@
-require "cryo/version"
+require 'colorize'
+require 'aws-sdk'
+require 'logger'
+require 'net/ntp'
+
+
+## require all ruby files recursively
+Dir.glob(File.join(File.dirname(__FILE__),'**/*.rb')).sort.each do |file|
+  require_relative file
+end
+
+
+class Cryo 
+
+  include Utils
+#  HOST = `hostname`.chomp!
+  attr_accessor :options, :s3, :md5, :sns, :logger, :key
+
+  def initialize(options={})
+    get_utc_timestamp # save start time for backup
+
+    self.options = options
+    self.logger = Logger.new(STDERR)
+    logger.level = Logger::DEBUG
+
+    @database = Database.create(options) \
+      unless options[:type] == 'list' or options[:type] == 'get'
+    @store = Store.create(options.merge(type: 's3',time: @time))
+    @message = Message.create(options.merge(type: 'sns'))
+    @snapshot_prefix = options[:snapshot_prefix]
+    @archive_prefix = options[:archive_prefix]
+    @key = get_timstamped_key_name
+    @snapshot_frequency = options[:snapshot_frequency]
+    @archive_frequency = options[:archive_frequency]
+    @snapshot_period = options[:snapshot_period]
+    @snapshot_bucket = options[:snapshot_bucket]
+    @archive_bucket = options[:archive_bucket]
+    @tmp_path = options[:tmp_path]
+  end
+
+
+  def backup!()
+    if @database.respond_to? 'get_gzipped_backup'
+      logger.info "getting compressed backup"
+      compressed_backup = @database.get_gzipped_backup
+      logger.info "got backup in #{(get_utc_time - @time).round 2} seconds"
+    else
+      logger.info "taking backup..."
+      backup_file = @database.get_backup
+      logger.info "got backup in #{(get_utc_time - @time).round 2} seconds"
+
+      timer = get_utc_time
+      logger.info "compressing backup..."
+      compressed_backup = gzip_file backup_file
+      logger.info "compressed backup in #{(get_utc_time - timer).round 2} seconds"
+    end
+
+    timer = get_utc_time
+    logger.info "storing backup..."
+    @store.put(content: Pathname.new(compressed_backup), bucket: options[:snapshot_bucket],key: @key)
+    logger.info "upload took #{(get_utc_time - timer).round 2} seconds"
+
+    logger.info "completed entire backup in #{(get_utc_time - @time).round 2} seconds :)"
+  end
+
+  def archive_and_purge()
+    logger.info "archiving and purging..."
+    @store.archive_and_purge()
+    logger.info "done archiving and purging :)"
+  end
+
+  def list_snapshots
+    snapshot_list = @store.get_bucket_listing(bucket: @snapshot_bucket, prefix: @snapshot_prefix)
+    puts "here is what I see in the snapshot bucket:"
+    snapshot_list.each { |i| puts "  #{i.key}"}
+  end
+
+  def list_archives
+    archive_list = @store.get_bucket_listing(bucket: @archive_bucket, prefix: @archive_prefix)
+    puts "here is what I see in the archive bucket:"
+    archive_list.each { |i| puts "  #{i.key}"}
+  end
+
+  def get_snapshot(snapshot)
+    basename = File.basename snapshot
+    puts "getting #{snapshot} and saving it in #{File.join(Dir.pwd,basename)}"
+    @store.get(bucket: @snapshot_bucket,key: snapshot,file: basename)
+  end
 
-module Cryo
-  # Your code goes here...
 end

data/lib/cryo/database.rb

@@ -0,0 +1,16 @@
+require_relative 'utils'
+
+class Database
+  include Utils
+
+  def get_backup()
+    raise "implement me"
+  end
+
+  class << self
+    def create(options={})
+      const_get(options[:type].to_s.capitalize).new(options)
+    end
+  end
+
+end

data/lib/cryo/database/mysql.rb

@@ -0,0 +1,44 @@
+# this has all of the logic to perform an entire dump of a remote rds host
+
+class Mysql < Database
+
+  include Utils
+  attr_accessor :user, :host, :password, :local_path, :tmp_path, :port
+
+  def initialize(opts={})
+    raise "you need to specify a password" unless opts[:password]
+    self.password = opts[:password]
+    raise "you need to specify a host" unless opts[:host]
+    self.host = opts[:host]
+    raise "you need to specify a tmp path" unless opts[:tmp_path]
+    self.tmp_path = opts[:tmp_path]
+    self.user = opts[:user] || 'ubuntu'
+    self.port = opts[:port] || '3306'
+    self.local_path = opts[:local_path] || get_tempfile
+    verify_system_dependency 'mysqldump'
+  end
+
+  ## run through all of the necessary steps to perform a backup
+  def get_backup()
+    get_dump
+    local_path
+  end
+  
+  def get_gzipped_backup
+    get_and_gzip_dump
+    local_path
+  end
+
+  private
+
+  ## perform a mysqldump to get an entire mysql dump on the local system, while gzipping it at the same time
+  def get_and_gzip_dump
+    safe_run "mysqldump --host=#{host} --user=#{user} --password=#{password} --all-databases --single-transaction | gzip > #{local_path}"
+  end
+
+  ## perform a mysqldump to get an entire mysql dump on the local system
+  def get_dump()
+    safe_run "mysqldump --host=#{host} --user=#{user} --password=#{password} --all-databases --single-transaction > #{local_path}"
+  end
+
+end

data/lib/cryo/database/postgres.rb

@@ -0,0 +1,31 @@
+# this has all of the logic to perform an entire dump of a remote postgress host
+
+class Postgres
+  include Utils
+  attr_accessor :user, :host, :password, :local_path, :tmp_path
+
+  def initialize(opts={})
+    raise "you need to specify a password" unless opts[:password]
+    self.password = opts[:password]
+    raise "you need to specify a host" unless opts[:host]
+    self.host = opts[:host]
+    raise "you need to specify a tmp path" unless opts[:tmp_path]
+    self.tmp_path = opts[:tmp_path]
+    self.user = opts[:user] || 'ubuntu'
+    self.local_path = opts[:local_path] || get_tempfile
+    verify_system_dependency 'pg_dumpall'
+  end
+
+  def get_backup()
+    take_dump
+  end
+
+  private
+
+  ## perform a pg_dumpall to get an entire pgdump on the local system
+  def take_dump()
+    safe_run "PGPASSWORD=#{password} pg_dumpall --host=#{host} --username=#{user} --file=#{local_path}"
+    local_path
+  end
+
+end

data/lib/cryo/database/redis.rb

@@ -0,0 +1,63 @@
+class Redis
+  include Utils
+  attr_accessor :user, :host, :remote_path, :local_path, :opts, :tmp_path
+
+  def initialize(opts={})
+    raise "you need to specify a remote host" unless opts[:host]
+    self.host = opts[:host]
+    self.user = opts[:user] || 'ubuntu'
+    raise "you need to specify a tmp path" unless opts[:tmp_path]
+    self.tmp_path = opts[:tmp_path]
+    self.remote_path = opts[:path] || '/mnt/redis/dump.rdb'
+    self.local_path = opts[:local_path] || get_tempfile
+  end
+
+
+  ## get a copy of the db from remote host
+  def get_backup()
+    take_dump
+  end
+
+
+  ## get a zipped copy of the db from remote host
+  def get_gzipped_backup
+    take_dump_and_gzip
+  end
+
+  private
+
+  ## copy the redis db into a new file and scp it here
+  def take_dump()
+    # TODO(martin): verify that both the local and remote hosts have enough free disk space for this to complete
+    temp_file = remote_path + "-backup-#{rand 99999}"
+    # this is kinda hacky, but we need to make sure that we remove a backup if we take one
+    begin
+      ssh "cp #{remote_path} #{temp_file}"
+      safe_run "scp #{user}@#{host}:#{temp_file} #{local_path}"
+    ensure
+      ssh "rm -f #{temp_file}"
+    end
+    local_path
+  end
+
+
+  ## copy the redis db into a new file and stream it here while zipping
+  def take_dump_and_gzip()
+    # TODO(martin): verify that both the local and remote hosts have enough free disk space for this to complete
+    temp_file = remote_path + "-backup-#{rand 99999}"
+    # this is kinda hacky, but we need to make sure that we remove a backup if we take one
+    begin
+      ssh "cp #{remote_path} #{temp_file}"
+      safe_run "(ssh #{user}@#{host} cat #{temp_file}) | gzip > #{local_path}"
+    ensure
+      ssh "rm -f #{temp_file}"
+    end
+    local_path
+  end
+
+
+  def ssh(command)
+    safe_run "ssh #{user}@#{host} #{command}"
+  end
+
+end

data/lib/cryo/message.rb

@@ -0,0 +1,23 @@
+class Message
+
+  def initialize(opts)
+  end
+
+  def get()
+    raise "implement me"
+  end
+
+  def put()
+    raise "implement me"
+  end
+
+
+  class << self
+    def create(options={})
+      message_class =  const_get(options[:type].to_s.capitalize)
+      return message_class.new(options)
+    end
+  end
+
+end
+

data/lib/cryo/message/sns.rb

@@ -0,0 +1,19 @@
+class Sns < Message
+  require 'aws-sdk'
+
+  def initialize(opts={})
+    AWS.config(:access_key_id => opts[:aws_access_key], 
+               :secret_access_key => opts[:aws_secret_key])
+    @sns = AWS::SNS::Client.new
+    @topic = opts[:topic] || opts[:topic_arn]
+  end
+
+
+  def send(opts={})
+    @sns.publish({
+                   :message => opts[:message],
+                   :subject => opts[:subject],
+                   :topic_arn => @topic
+                 })
+  end
+end

data/lib/cryo/store.rb

@@ -0,0 +1,139 @@
+#require 'net/ntp'
+require 'logger'
+
+class Store
+
+  include Utils
+  attr_accessor :logger
+  
+  def initialize(opts={})
+    self.logger = Logger.new(STDERR)
+    logger.level = Logger::DEBUG
+
+    @snapshot_frequency = opts[:snapshot_frequency]
+    @archive_frequency = opts[:archive_frequency]
+    @snapshot_period = opts[:snapshot_period]
+    @snapshot_prefix = opts[:snapshot_prefix]
+    @archive_prefix = opts[:archive_prefix]
+    @time = opts[:time]
+  end
+
+  def get()
+    raise "implement me"
+  end
+
+  def put()
+    raise "implement me"
+  end
+
+  def get_snapshot_list()
+    raise "implement me"
+  end
+
+  def get_archive_list()
+    raise "implement me"
+  end
+
+  class << self
+    def create(options={})
+      const_get(options[:type].to_s.capitalize).new(options)
+    end
+  end
+  
+  def archive_and_purge
+    snapshot_list = get_snapshot_list
+    newest_archive = get_newest_archive
+    recursive_archive_and_purge(snapshot_list: snapshot_list, newest_archive: newest_archive)
+  end
+
+  protected
+
+  def get_newest_archive()
+    raise "implment me"
+  end
+
+
+  def recursive_archive_and_purge(opts={})
+    logger.debug "entering recursive_archive_and_purge"
+    snapshot_list = opts[:snapshot_list]
+
+    # return if there are no snapshots
+    if snapshot_list.empty?
+      logger.info "no snapshots found"
+      return true 
+    end
+
+    # return if there are not enough snapshots avilable
+    minium_number_of_snapshots = (@snapshot_period.to_f/@snapshot_frequency.to_f).ceil
+    if snapshot_list.size < minium_number_of_snapshots
+      logger.info "not enough snapshots avilable for archiving"
+      logger.info "we found #{snapshot_list.size} but we need to keep at least #{minium_number_of_snapshots}"
+      return true
+    end
+
+    oldest_snapshot = snapshot_list.shift
+    oldest_snapshot_age = get_age_from_key_name(oldest_snapshot)
+
+    logger.debug "oldest_snapshot is #{oldest_snapshot}"
+    logger.debug "oldest_snapshot_age is #{oldest_snapshot_age}"
+
+    # return if the oldest snapshot it not old enough to be archived
+    if oldest_snapshot_age < @snapshot_period
+      logger.info "all snapshots are younger than snapshot_period"
+      return true 
+    end
+
+    # if we got this far, then the oldest snapshot needs to be either archived or deleted
+    newest_archive = get_newest_archive
+
+    # check to see if we have any archives
+    if newest_archive.empty?
+      logger.info "looks like we don't have any archives yet"
+      logger.info "archiving oldest snapshot #{oldest_snapshot}"
+      archive_snapshot oldest_snapshot
+      logger.debug "recursing..."
+      recursive_archive_and_purge(snapshot_list: snapshot_list, newest_archive: oldest_snapshot)
+      return true
+    end
+
+
+    newest_archive_age = get_age_from_key_name(newest_archive)
+
+    # check to see if the oldest snapshot should be archived
+    if need_to_archive?(oldest_snapshot_age,newest_archive_age)
+      logger.info "archiving oldest snapshot #{oldest_snapshot}"
+      archive_snapshot oldest_snapshot
+      logger.debug "recursing..."
+      recursive_archive_and_purge(snapshot_list: snapshot_list, newest_archive: oldest_snapshot)
+      return true
+    end
+
+    # check the next oldest snapshot too, before we throw this one away
+    second_oldest_snapshot = opts[:snapshot_list].first
+    second_oldest_snapshot_age = get_age_from_key_name(second_oldest_snapshot)
+
+    if need_to_archive?(second_oldest_snapshot_age,newest_archive_age)
+      logger.info "archiving oldest snapshot #{oldest_snapshot}"
+      archive_snapshot oldest_snapshot
+      logger.debug "recursing..."
+      recursive_archive_and_purge(snapshot_list: snapshot_list, newest_archive: oldest_snapshot)
+      return true
+    end
+
+    # if we got this far, then we just need to delete the oldest snapshot
+    logger.info "deleting oldest snapshot #{oldest_snapshot}"
+    delete_snapshot oldest_snapshot
+    logger.debug "recursing"
+    recursive_archive_and_purge(snapshot_list: snapshot_list, newest_archive: newest_archive)
+    return true
+  end
+  
+  def archive_snapshot
+    raise "implment me"
+  end
+
+  def delete
+    raise "implment me"
+  end
+
+end

data/lib/cryo/store/s3.rb

@@ -0,0 +1,128 @@
+class S3 < Store
+  require 'aws-sdk'
+  
+  attr_accessor :snapshot_bucket, :archive_bucket, :prefix
+
+  def initialize(opts={})
+    super(opts)
+    AWS.config(:access_key_id => opts[:aws_access_key], 
+               :secret_access_key => opts[:aws_secret_key])
+    @s3 = AWS::S3.new
+    @snapshot_bucket = @s3.buckets[opts[:snapshot_bucket]]
+    @archive_bucket = @s3.buckets[opts[:archive_bucket]]
+  end
+
+  def get(opts={})
+    bucket = opts[:bucket]
+    key = opts[:key]
+    file_path = opts[:file] || opts[:path]
+    if file_path
+      File.open(file_path,'w') do |file|
+        @s3.buckets[bucket].objects[key].read {|chunk| file.write chunk}
+        return true
+      end
+    else
+      return @s3.buckets[bucket].objects[key].read
+    end
+  end
+
+  
+  def put(opts={})
+    bucket = opts[:bucket]
+    key = opts[:key]
+    content = opts[:content]
+    @s3.buckets[bucket].objects[key].write(content) # TODO: verify that bucket exists?
+  end
+
+
+  def etag(opts={})
+    bucket = opts[:bucket]
+    key = opts[:key]
+    @s3.buckets[bucket].objects[key].etag
+  end
+
+
+  # retun an array listing the objects in our snapshot bucket
+  def get_snapshot_list
+    get_bucket_listing(bucket: @snapshot_bucket, prefix: @prefix)
+  end
+
+ 
+  # retun an array listing the objects in our archive bucket
+  def get_archive_list
+    get_bucket_listing(bucket: archive_bucket, prefix: @prefix)
+  end
+
+  # return an array listing of objects in a bucket
+  def get_bucket_listing(opts={})
+    bucket = opts[:bucket]
+    prefix = opts[:prefix]
+    list = []
+    @s3.buckets[bucket].objects.with_prefix(prefix).each do |object|
+      list << object
+    end
+    list
+  end
+
+  def get_snapshot_list
+    snapshot_list = []
+    @snapshot_bucket.objects.with_prefix(@snapshot_prefix).each do |object|
+      snapshot_list << trim_snapshot_name(object.key)
+    end
+    snapshot_list
+  end
+
+  protected
+
+  def expand_snapshot_name(shortname)
+    @snapshot_prefix + shortname + "Z.cryo"
+  end
+
+  def expand_archive_name(shortname)
+    @archive_prefix + shortname + "Z.cryo"
+  end
+
+  def trim_snapshot_name(longname)
+    longname.gsub(/^#{@snapshot_prefix}/,'').gsub(/Z\.cryo$/,'')
+  end
+
+  def trim_archive_name(longname)
+    return "" if longname.nil?
+    longname.gsub(/^#{@archive_prefix}/,'').gsub(/Z\.cryo$/,'')
+  end
+
+  def delete_snapshot(snapshot)
+    full_snapshot_name = expand_snapshot_name(snapshot)
+    @snapshot_bucket.objects[full_snapshot_name].delete
+  end
+
+  def archive_snapshot(snapshot)
+    logger.info "archiving snapshot #{snapshot}"
+    full_snapshot_name = expand_snapshot_name(snapshot)
+    full_archive_name = expand_archive_name(snapshot)
+    logger.debug "full_snapshot_name is #{full_snapshot_name}"
+    logger.debug "full_archive_name is #{full_archive_name}"
+    snapshot_object = @snapshot_bucket.objects[full_snapshot_name]
+    # if we have already copied the object, just delete the snapshot
+    if @archive_bucket.objects[full_archive_name].exists?
+      snapshot_object.delete
+    else
+      snapshot_object.move_to(full_archive_name, :bucket => @archive_bucket)
+    end
+  end
+  
+
+  # this function returns the last item in a bucket that matches the given prefix
+  def get_newest_archive(prefix=@archive_prefix)
+    tree = @archive_bucket.objects.with_prefix(prefix).as_tree
+    directories = tree.children.select(&:branch?).collect(&:prefix)
+    if directories.empty?
+      matches = []
+      @archive_bucket.objects.with_prefix(prefix).each {|o| matches << o.key}
+      return trim_archive_name(matches.last)
+    else
+      # recurse
+      get_newest_archive(directories.last)
+    end
+  end
+end

data/lib/cryo/utils.rb

@@ -0,0 +1,117 @@
+module Utils
+
+  require 'zlib'
+  require 'net/ntp'
+  require 'fileutils'
+    
+
+  def delete_file(path)
+    File.delete(path) if File.exists?(path)
+  end
+
+
+  def get_tempfile
+    #    Tempfile.new('redis-backup','/mnt/cryo').path
+    tmp_file = File.join(@tmp_path,"tmp-#{rand 9999}")
+    at_exit {delete_file tmp_file}
+    FileUtils.touch tmp_file
+    tmp_file
+  end
+
+
+  def ungzip_file(path)
+    # get a temp file
+    tempfile = get_tempfile
+    #logger.info "unzipping #{path} to #{tempfile}..."
+    
+    # stream the gzipped file into an uncompressed file
+    Zlib::GzipReader.open(path) do |gz|
+      File.open(tempfile,'w') do |open_file|
+        # write 1M chunks at a time
+        open_file.write gz.read(1024*1024) until gz.eof?
+      end
+    end
+    #logger.info "finished unzipping file"
+
+    # return unzipped file
+    tempfile
+  end
+
+
+  def gzip_file(path)
+    # given a path to a file, return a gzipped version of it
+    tempfile = get_tempfile
+    #logger.info "gzipping #{path} to #{tempfile}"
+
+    # stream the gzipped content into a file as we compute it
+    Zlib::GzipWriter.open(tempfile) do |gz|
+      File.open(path) do |f|
+        # write 1M chunks at a time
+        gz.write f.read(1024*1024) until f.eof?
+      end
+    end
+    #logger.info "done unzipping"
+    tempfile
+  end
+
+  def safe_run(command)
+    #logger.debug "about to run #{command}"
+    output = `bash -c "set -o pipefail && #{command}"`.chomp
+    raise "command '#{command}' failed!\nOutput was:\n#{output}" unless $?.success?
+    true
+  end
+
+  def verify_system_dependency(command)
+    raise "system dependency #{command} is not unstalled" unless system "which #{command} > /dev/null"
+  end
+
+  def get_utc_time
+    retries = 5
+    begin
+      Net::NTP.get("us.pool.ntp.org").time.getutc
+    rescue Object => o
+      retries -= 1
+      if retries > 0
+        logger.debug "retrying ntp query again..."
+        sleep 2
+        retry
+      end
+      throw o
+    end
+  end
+
+  def get_utc_timestamp()
+    @time ||= get_utc_time  # don't change the endpoint!!! 
+    @timestamp ||= @time.strftime("%Y/%m/%d/%H:%M:%S")
+  end
+
+  def get_timstamped_key_name()
+    "#{@snapshot_prefix}#{@timestamp}Z.cryo"
+  end
+
+  def get_utc_time_from_key_name(key_name)
+    logger.debug "getting time for #{key_name}"
+    year,month,day,time = key_name.split('/')
+    hour,min,sec = time.split(':')
+    Time.utc(year,month,day,hour,min,sec) 
+  end
+
+  # returns the age of the snapshot in mins
+  def get_age_from_key_name(key_name)
+    snapshot_time = get_utc_time_from_key_name(key_name)
+    age_in_mins_as_float = (@time - snapshot_time) / 60
+    age_in_mins_as_int = age_in_mins_as_float.to_i
+  end
+
+  # find out if we have an archive that is more recent than the snapshot period
+  def need_to_archive?(old_snapshot_age,new_archive_age)
+    logger.debug "checking to see if we should archive"
+    logger.debug "oldest snapshot age is #{old_snapshot_age}"
+    logger.debug "newest archive time is #{new_archive_age}"
+    logger.debug "@snapshot_period is #{@archive_frequency}"
+    answer = (new_archive_age - old_snapshot_age) > @archive_frequency
+    logger.debug "returning #{answer.inspect}"
+    return answer
+  end
+
+end

data/lib/cryo/version.rb

@@ -1,3 +1,3 @@
-module Cryo
-  VERSION = "0.0.1"
+class Cryo
+  VERSION = "0.0.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cryo
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,22 +9,118 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-06 00:00:00.000000000 Z
-dependencies: []
+date: 2013-04-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: net-ntp
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - nathan.baxter@airbnb.com
-executables: []
+executables:
+- cryo
 extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
 - Gemfile
 - LICENSE
+- Makefile
 - README.md
 - Rakefile
+- bin/bootstrap_cryo.sh
+- bin/cryo
+- bin/test
+- bin/test2
 - cryo.gemspec
+- examples/rds.sh
 - lib/cryo.rb
+- lib/cryo/database.rb
+- lib/cryo/database/mysql.rb
+- lib/cryo/database/postgres.rb
+- lib/cryo/database/redis.rb
+- lib/cryo/message.rb
+- lib/cryo/message/sns.rb
+- lib/cryo/store.rb
+- lib/cryo/store/s3.rb
+- lib/cryo/utils.rb
 - lib/cryo/version.rb
 homepage: https://github.com/airbnb/cryo
 licenses: []
@@ -49,5 +145,6 @@ rubyforge_project:
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
-summary: Tool for snapshotting data, backing it up, verifying it, and cycling it.
+summary: Tool for snapshotting data, backing it up, verifying it, cycling it, and
+  triggering notifications.
 test_files: []