appshot 0.0.1

data/.gitignore

@@ -0,0 +1,21 @@
+*.gem
+*.rbc
+.DS_Store
+.bundle
+.config
+.rvmrc
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+html
+lib/bundler/man
+pkg
+rdoc
+results.html
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,19 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in appshot.gemspec
+gemspec
+
+group :development do
+  gem "awesome_print"
+  gem "fivemat"
+  gem "growl"
+  gem "guard"
+  gem "guard-bundler"
+  gem "guard-rspec"
+  gem "guard-cucumber"
+  gem "rb-fsevent"
+  gem "ruby_gntp"
+  gem "rb-readline"
+  gem "debugger", :platforms => :mri_19 if RUBY_VERSION >= "1.9.3"
+  gem "yard"
+end

data/Guardfile

@@ -0,0 +1,19 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'bundler' do
+  watch('Gemfile')
+  watch(/^.+\.gemspec/)
+end
+
+guard 'rspec', :version => 2, :cli=> "--format Fivemat --debugger" do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+
+guard 'cucumber' do
+  watch(%r{^features/.+\.feature$})
+  watch(%r{^features/support/.+$})          { 'features' }
+  watch(%r{^features/step_definitions/(.+)_steps\.rb$}) { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'features' }
+end

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Morgan Nelson
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Appshot
+
+AppShot takes consistent snapshots of your server volumes using pluggable modules for applications, filesystems, and storage providers. One issue with taking snapshots of storage volumes is that without fundamental orchestration at both the application and filesystem level, you are almost certainly not going to get a consistent snapshot.  In-flight data in application and kernel caches will likely be missed, and while you think you are making good backups of your data, when the time comes, you realize that it doesn't work.
+
+The goal for AppShot is to provide a framework for creating pluggable providers for application pausing/restarting, filesystem freezeing/thawing and volume management, whether local (LVM, et. al.) or cloud (Amazon EBS, et. al.).  The software isn't quite there yet, but progress is being made.
+
+Currently only supports some Device Mapper filesystems for freezing (ext4, XFS), Mysql as an application, and Amazon EBS as the volume store.  More on the way.
+
+Support for ext3, ReiserFS and JFS in filesystem freezing should be a trivial method alias, but I haven't had time to test them.  Filesystem freezing is unnecessary when using LMV2's volume snapshot capability (the underlying device-mapper provides this functionality for free), but I haven't implemented LVM2 snapshots yet.
+
+As always, patches welcome.  I don't have much access or experience with anything but linux or OS X, so patches for any flavor of BSD or (Open)Solaris gratefully accepted.
+
+## Installation
+
+  $ gem install appshot
+
+## Usage
+
+Setup your appshot.conf file and run appshot.
+
+    $ appshot
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/README.rdoc

@@ -0,0 +1,20 @@
+= appshot - DESCRIBE YOUR GEM
+
+Author::  Morgan Nelson (morgan\.nelson@gmail.com)
+Copyright:: Copyright (c) 2012 Morgan Nelson
+
+
+License:: mit, see LICENSE.txt
+
+
+
+Appshot coordinates application, filesystem, and storage volume operations to give you consistent snapshots.
+
+== Links
+
+* {Source on Github}[http://github.com/korishev/appshot]
+
+== Install
+
+gem install appshot
+

data/Rakefile

@@ -0,0 +1,44 @@
+require 'bundler'
+require 'rake/clean'
+
+require 'rspec/core/rake_task'
+
+require 'cucumber'
+require 'cucumber/rake/task'
+gem 'rdoc' # we need the installed RDoc gem, not the system one
+require 'rdoc/task'
+
+include Rake::DSL
+
+Bundler::GemHelper.install_tasks
+
+
+RSpec::Core::RakeTask.new do |t|
+  # Put spec opts in a file named .rspec in root
+end
+
+
+CUKE_RESULTS = 'results.html'
+CLEAN << CUKE_RESULTS
+Cucumber::Rake::Task.new(:features) do |t|
+  t.cucumber_opts = "features --format html -o #{CUKE_RESULTS} --format pretty --no-source -x"
+  t.fork = false
+end
+
+Rake::RDocTask.new do |rd|
+
+  rd.main = "README.rdoc"
+
+  rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
+end
+
+task :default => [:spec,:features]
+
+
+desc "Open an irb session preloaded with Appshot"
+
+task :console do
+  sh "irb -rrubygems -I lib -r appshot.rb"
+end
+
+task :c => :console

data/appshot.conf.example

@@ -0,0 +1,7 @@
+appshot "mysql_userdb" do
+  mysql name: "userdb", port: 1536, username: "pooky", password: "bear"
+  xfs mount_point: "/mnt/untitled"
+  ebs_snapshot volume_id: "vol-some_id", aws_access_key_id: "BOOM", aws_secret_access_key: "KEEP_IT_SECRET"
+  ebs_prune volume_id: "vol-some_id", snapshots_to_keep: 3, minimum_retention_days: 0, aws_access_key_id: "BOOM", aws_secret_access_key: "KEEP_IT_SECRET"
+end
+

data/appshot.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/appshot/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Morgan Nelson"]
+  gem.email         = ["mnelson@steele-forge.com"]
+  gem.description   = %q{AppShot takes an application aware snapshot of your drive volume (one day) using pluggable modules representing actions to be taken concerning applications, filesystems, cloud providers, etc.}
+  gem.summary       = %q{Appshot takes consistent snapshots of drive volumes.}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "appshot"
+  gem.require_paths = ["lib"]
+  gem.version       = Appshot::VERSION
+
+  gem.add_runtime_dependency "fog", "~> 1.4.0"
+  gem.add_runtime_dependency "methadone", "~> 1.2.1"
+  gem.add_runtime_dependency "open4", "~> 1.3.0"
+  gem.add_runtime_dependency "mysql2", "~> 0.3.11"
+
+  gem.add_development_dependency "aruba"
+  gem.add_development_dependency "bundler",      "~> 1.0"
+  gem.add_development_dependency "fabrication",  "~> 2.0.1"
+  gem.add_development_dependency "pry"
+  gem.add_development_dependency "rake",         "~> 0.9.0"
+  gem.add_development_dependency "rake",         "~> 0.9.2"
+  gem.add_development_dependency "rdoc"
+  gem.add_development_dependency "rspec",        "~> 2.6"
+  gem.add_development_dependency "timecop",      "~> 0.3.5"
+end

data/bin/appshot

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'methadone'
+require 'appshot'
+
+class App
+  include Methadone::Main
+  include Methadone::CLILogging
+
+  main do |*appshot_names|
+    config_file_name = options["config_file"] || "/etc/appshot/appshot.cfg"
+    args = { :appshot_names => appshot_names }
+
+    appshot = Appshot.new(File.read(config_file_name))
+
+    appshot.run_pass(options, args)
+  end
+
+  version Appshot::VERSION
+  description "Use Appshot to take consistent snapshots of your data volumes"
+
+  use_log_level_option
+
+  on("-c CONFIG_FILE","--config_file", "Configuration File", /\S+/)
+  on("-l", "--list_appshots", "--list-appshots", "List known appshots", /\S+/)
+  on("-p", "Pretend mode", /\S+/)
+
+  arg :appshot_names, :any, "The name of one or more appshots as defined in your config file"
+
+  go!
+end

data/features/appshot.feature

@@ -0,0 +1,55 @@
+Feature: Appshot CLI mode
+  In order to take consistent snapshots of my data volumes
+  I want to check my application parameters
+  So I have the options I need
+
+  Scenario: App displays help when asked
+    When I get help for "appshot"
+    Then the exit status should be 0
+    And the banner should be present
+    And the banner should document that this app takes options
+    And the following options should be documented:
+      |--version      |
+      |-c             |
+      |--config_file  |
+      |-l             |
+      |--list_appshots|
+      |-p             |
+    And the banner should document that this app's arguments are:
+      |appshot_names...| optional|
+
+  Scenario: User asks for list of appshots from empty file
+    Given an empty file named "/tmp/empty.cfg"
+    When I successfully run `appshot -c /tmp/empty.cfg --list`
+    Then the output should contain "There are no appshots configured"
+
+  Scenario: User asks for list of appshots from file with one appshot
+    Given a file named "/tmp/one_appshot.cfg" with:
+    """
+    appshot "mysql_userdb" do
+      mysql name: "userdb", port: 1536, username: "pooky"
+      ebs_snapshot volume_id: "vol-4ed40599", aws_access_key_id: "BOO", aws_secret_access_key: "GAH"
+      ebs_prune snapshots_to_keep: 15, minimum_retention_days: 5
+    end
+    """
+    When I successfully run `appshot -c /tmp/one_appshot.cfg --list`
+    Then the output should contain "There is one appshot configured"
+
+  Scenario: User asks for list of appshots from file with multiple appshots
+    Given a file named "/tmp/one_appshot.cfg" with:
+    """
+    appshot "mysql_userdb" do
+      mysql name: "userdb", port: 1536, username: "pooky"
+      ebs_snapshot volume_id: "vol-4ed40599", aws_access_key_id: "BOO", aws_secret_access_key: "GAH"
+      ebs_prune snapshots_to_keep: 15, minimum_retention_days: 5
+    end
+
+    appshot "mysql_datadb" do
+      mysql name: "datadb", port: 1536, username: "pooky"
+      ebs_snapshot volume_id: "vol-ded06738", aws_access_key_id: "BOO", aws_secret_access_key: "GAH"
+      ebs_prune snapshots_to_keep: 15, minimum_retention_days: 5
+    end
+    """
+    When I successfully run `appshot -c /tmp/one_appshot.cfg --list`
+    Then the output should contain "There are 2 appshots configured"
+

data/features/step_definitions/appshot_steps.rb

@@ -0,0 +1,3 @@
+When 'I debug' do
+  binding.pry
+end

data/features/support/env.rb

@@ -0,0 +1,18 @@
+require 'aruba/cucumber'
+require 'methadone/cucumber'
+require 'debugger'
+require 'pry'
+
+ENV['PATH'] = "#{File.expand_path(File.dirname(__FILE__) + '/../../bin')}#{File::PATH_SEPARATOR}#{ENV['PATH']}"
+LIB_DIR = File.join(File.expand_path(File.dirname(__FILE__)),'..','..','lib')
+
+Before do
+    # Using "announce" causes massive warnings on 1.9.2
+  @puts = true
+  @original_rubylib = ENV['RUBYLIB']
+  ENV['RUBYLIB'] = LIB_DIR + File::PATH_SEPARATOR + ENV['RUBYLIB'].to_s
+end
+
+After do
+  ENV['RUBYLIB'] = @original_rubylib
+end

data/lib/appshot/app/mysql.rb

@@ -0,0 +1,43 @@
+require 'mysql2'
+require 'methadone'
+
+class Appshot
+  class Mysql
+    include Methadone::CLILogging
+
+    def initialize(opts={})
+      @host = opts[:hostname] || "localhost"
+      @name = opts[:name]     || "mysql"
+      @port = opts[:port]     || 3306
+      @user = opts[:username] || "mysql"
+      @password = opts[:password]
+    end
+
+    def call(call_chain)
+      next_action = call_chain.shift
+      @client = Mysql2::Client.new(username: @user, hostname: @host, password: @password, port: @port, database: @name)
+
+      lock
+      next_action.call(call_chain) unless next_action.nil?
+      unlock
+    end
+
+    private
+
+    def lock
+      begin
+        Timeout::timeout(20) do
+          @client.query("FLUSH TABLES WITH READ LOCK")
+        end
+      rescue
+        error "Could not get MySQL tables flushed and locked before timeout."
+        @client.query("UNLOCK TABLES")
+        @client.close
+      end
+    end
+
+    def unlock
+      @client.query("UNLOCK TABLES")
+    end
+  end
+end

data/lib/appshot/app.rb

@@ -0,0 +1 @@
+require_relative 'app/mysql'

data/lib/appshot/filesystem/dm.rb

@@ -0,0 +1,53 @@
+require 'pathname'
+
+class Appshot
+  class DM
+    def initialize(opts={})
+      @mount_point = Pathname.new(opts[:mount_point] || "")
+      validate_mount_point(@mount_point)
+    end
+
+    def call(call_chain=[])
+      next_action = call_chain.shift
+
+      freeze
+      next_action.call(call_chain) unless next_action.nil?
+      unfreeze
+    end
+
+    private
+
+    def validate_mount_point(mount_point)
+      raise "Your mount point: '#{mount_point}' is not a mount point!" unless @mount_point.mountpoint?
+      raise "We cannot currently unfreeze the root filesystem, leaving your system unusable. Aborting." if @mount_point.to_s == "/"
+    end
+
+    def mount_point
+      @mount_point.to_s
+    end
+
+    def freeze_command
+      %x{ /sbin/fsfreeze -f #{mount_point} }
+    end
+
+    def unfreeze_command
+      %x{ /sbin/fsfreeze -u #{mount_point} }
+    end
+
+    def freeze
+      begin
+        freeze_command
+      rescue
+        puts "There was a problem freezing your mount point: #{mount_point}"
+      end
+    end
+
+    def unfreeze
+      unfreeze_command
+    end
+
+    def check_mount_point
+      mounts = %x{ df }
+    end
+  end
+end

data/lib/appshot/filesystem.rb

@@ -0,0 +1 @@
+require_relative 'filesystem/dm'

data/lib/appshot/version.rb

@@ -0,0 +1,3 @@
+class Appshot
+  VERSION = "0.0.1"
+end

data/lib/appshot/volume/ebs_prune.rb

@@ -0,0 +1,38 @@
+class Appshot
+  class EBS_Prune
+    def initialize(opts={})
+      @volume_id = opts[:volume_id] || ""
+      @snapshots_to_keep = opts[:snapshots_to_keep] || 5
+      @minimum_retention_days = opts[:minimum_retention_days] || 3
+      @region                = opts[:region] unless opts[:region].nil?
+      @aws_secret_access_key = opts[:aws_secret_access_key] || ""
+      @aws_access_key_id     = opts[:aws_access_key_id] || ""
+    end
+
+    def call(call_chain=[])
+      next_action = call_chain.shift
+      prune_snapshots if valid?
+      next_action.call(call_chain) unless next_action.nil?
+    end
+
+    def valid?
+      raise ArgumentError.new "volume_id must be specified for an ebs_prune" if @volume_id.empty?
+      raise ArgumentError.new "aws_access_key_id must be specified for an ebs_prune" if @aws_access_key_id.empty?
+      raise ArgumentError.new "aws_secret_access_key must be specified for an ebs_prune" if @aws_secret_access_key.empty?
+      true
+    end
+
+    private
+
+    def ebs_volume
+      options = { aws_access_key_id: @aws_access_key_id, aws_secret_access_key: @aws_secret_access_key }
+      options.update( { region: @region } ) if @region
+      Appshot::EBS_Volume.new(options)
+    end
+
+    def prune_snapshots
+      not_after_time = Time.now - (@minimum_retention_days * 86400)
+      ebs_volume.prune_snapshots(@volume_id, snapshots_to_keep: @snapshots_to_keep, not_after_time: not_after_time )
+    end
+  end
+end

data/lib/appshot/volume/ebs_snapshot.rb

@@ -0,0 +1,34 @@
+require_relative 'ebs_volume'
+
+class Appshot
+  class EBS_Snapshot
+    def initialize(opts={})
+      @volume_id             = opts[:volume_id] || ""
+      @region                = opts[:region] unless opts[:region].nil?
+      @aws_secret_access_key = opts[:aws_secret_access_key] || ""
+      @aws_access_key_id     = opts[:aws_access_key_id] || ""
+      @description           = opts[:description] || "Standard Snapshot #{Time.now.utc}"
+    end
+
+    def call(call_chain=[])
+      next_action = call_chain.shift
+      ebs_volume.snap(@volume_id, @description) if valid?
+      next_action.call(call_chain) unless next_action.nil?
+    end
+
+    def valid?
+      raise ArgumentError.new "volume_id must be specified for an ebs_snapshot" if @volume_id.empty?
+      raise ArgumentError.new "aws_access_key_id must be specified for an ebs_snapshot" if @aws_access_key_id.empty?
+      raise ArgumentError.new "aws_secret_access_key must be specified for an ebs_snapshot" if @aws_secret_access_key.empty?
+      true
+    end
+
+private
+
+    def ebs_volume
+      options = { aws_access_key_id: @aws_access_key_id, aws_secret_access_key: @aws_secret_access_key }
+      options.update( { region: @region } ) if @region
+      Appshot::EBS_Volume.new(options)
+    end
+  end
+end

data/lib/appshot/volume/ebs_volume.rb

@@ -0,0 +1,65 @@
+require 'fog'
+require 'awesome_print'
+
+class Appshot
+  class EBS_Volume
+    include Fog
+
+    attr_accessor :volume_id, :region
+    attr_writer   :aws_access_key_id, :aws_secret_access_key
+    attr_reader   :provider
+
+
+    def initialize(opts = {})
+      @provider = opts[:provider] || 'AWS'
+      @region = opts[:region] || "us-east-1"
+      @aws_access_key_id = opts[:aws_access_key_id] unless opts[:aws_access_key_id].nil?
+      @aws_secret_access_key = opts[:aws_secret_access_key] unless opts[:aws_secret_access_key].nil?
+
+      @fog = Fog::Compute.new(:provider => @provider,
+                              :region => @region,
+                              :aws_access_key_id => @aws_access_key_id,
+                              :aws_secret_access_key => @aws_secret_access_key,
+                             )
+    end
+
+    def volumes
+      @fog.volumes.all
+    end
+
+    def snap(volume_id, description = "", options = {})
+      snap = @fog.snapshots.create({:volume_id => volume_id, :description => description}.merge(options))
+      while snap.state != "completed" do
+        sleep 0.5
+        snap.reload
+      end
+      snap
+    end
+
+    def snapshots_for(volume_id)
+      # for more info on Amazon EC2 filters (like "volume-id" below) please
+      # refer to http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeVolumes.html
+      @fog.snapshots.all("volume-id" => volume_id)
+    end
+
+    #def prune_snapshots(volume_id, snapshots_to_keep, not_after_time = Time.now)
+    def prune_snapshots(volume_id, options = {})
+      snapshots_to_keep  = options[:snapshots_to_keep] || 3
+      not_after_time     = options[:not_after_time] || Time.now
+      snapshots          = snapshots_for(volume_id)
+
+      (snapshots.count - snapshots_to_keep).times do
+        snapshots.first.destroy if snapshots.first.created_at < not_after_time
+        snapshots.reload
+      end
+    end
+
+    def day_offset(days)
+      return days if days.nil?
+      Time.now - (days * 86400)
+    end
+    def days_to_seconds(days)
+      days * 86400
+    end
+  end
+end

data/lib/appshot/volume.rb

@@ -0,0 +1,7 @@
+require_relative 'volume/ebs_volume'
+require_relative 'volume/ebs_snapshot'
+require_relative 'volume/ebs_prune'
+
+class Volume
+
+end