gigpark-ec2onrails 0.9.10.3

data/server/files/usr/local/ec2onrails/bin/archive_file

@@ -0,0 +1,44 @@
+#!/usr/bin/ruby
+
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# This script archives a file to S3
+
+require "rubygems"
+require "optiflag"
+require "#{File.dirname(__FILE__)}/../lib/s3_helper"
+require "#{File.dirname(__FILE__)}/../lib/utils"
+
+include FileUtils
+
+module CommandLineArgs extend OptiFlagSet
+  optional_flag "bucket"
+  optional_flag "dir"
+  optional_flag "file"
+  and_process!
+end
+
+# include the hostname in the bucket name so test instances don't accidentally clobber real backups
+bucket = ARGV.flags.bucket
+dir = ARGV.flags.dir
+file = ARGV.flags.file
+exit unless File.exists?(file)
+
+@s3 = Ec2onrails::S3Helper.new(bucket, dir)
+@s3.store_file(file)

data/server/files/usr/local/ec2onrails/bin/backup_app_db

@@ -0,0 +1,160 @@
+#!/usr/bin/ruby
+      
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    
+#exit if the current application is not deployed, meaning there is nothing to back up!
+exit unless File.exists?("/mnt/app/current")
+  
+require "rubygems"
+require "optiflag"
+require "fileutils"
+require 'EC2'
+require "#{File.dirname(__FILE__)}/../lib/mysql_helper"
+require "#{File.dirname(__FILE__)}/../lib/s3_helper"
+require "#{File.dirname(__FILE__)}/../lib/aws_helper"
+require "#{File.dirname(__FILE__)}/../lib/roles_helper"
+
+require "#{File.dirname(__FILE__)}/../lib/utils"
+
+# Only run if this instance is the db_pimrary
+# The original code would run on any instance that had /etc/init.d/mysql
+# Which was pretty much all instances no matter what role
+include Ec2onrails::RolesHelper
+exit unless in_role?(:db_primary)
+
+    
+module CommandLineArgs extend OptiFlagSet
+  optional_flag "bucket"  
+  optional_flag "dir"
+  optional_switch_flag "incremental"
+  optional_switch_flag "reset"
+  optional_switch_flag "no_ebs"
+  and_process!
+end
+@mysql = Ec2onrails::MysqlHelper.new
+
+
+if File.exists?("/etc/mysql/conf.d/mysql-ec2-ebs.cnf")  && !ARGV.flags.no_ebs
+  # we have ebs enabled....
+        
+  @aws   = Ec2onrails::AwsHelper.new
+  vols = YAML::load(File.read("/etc/ec2onrails/ebs_info.yml"))
+  ec2 = EC2::Base.new( :access_key_id => @aws.aws_access_key, :secret_access_key => @aws.aws_secret_access_key )
+  
+  #lets make sure we have space: AMAZON puts a 500 limit on the number of snapshots
+  snaps = ec2.describe_snapshots['snapshotSet']['item'] rescue nil
+  if snaps && snaps.size > 450
+    # TODO:
+    # can we make this a bit smarter?  With a limit of 500, that is difficult.  
+    # possible ideas (and some commented out code below)
+    #  * only apply cleanups to the volume_ids attached to this instance
+    #  * keep the last week worth (at hrly snapshots), then daily for a month, then monthly
+    #  * a sweeper task 
+    #
+    # vol_ids = []
+    # vols.each_pair{|k,v| vol_ids << v['volume_id']}
+    # #lets only work on those that apply for these volumnes attached 
+    # snaps = snaps.collect{|sn| vol_ids.index(sn['volumeId']) ? sn : nil}.compact
+    # # get them sorted
+    snaps = snaps.sort_by{|snapshot| snapshot['startTime']}.reverse
+    curr_batch = {}
+    remaining = []
+    snaps[200..-1].each do |sn| 
+      next if sn.blank? || sn['status'] != 'completed'
+      today = Date.parse(sn['startTime']).to_s
+      if curr_batch[sn['volumeId']] != today
+        curr_batch[sn['volumeId']] = today
+        remaining << sn
+      else
+        ec2.delete_snapshot(:snapshot_id => sn['snapshotId'])
+      end
+      # next unless vol_ids.index(sn['volumeId'])
+    end
+    if remaining.size > 400
+      puts "  WARNING: still contains #{remaining.size} snapshots; removing the oldest 100 to clean up space"
+      remaining[350..-1].each do |sn|
+        ec2.delete_snapshot(:snapshot_id => sn['snapshotId'])
+      end
+    end
+  else
+    puts "Could not retrieve snapshots: auto archiving cleanup will not occur" unless snaps
+  end
+  
+  @mysql.execute do |conn|
+    begin
+      conn.query "FLUSH TABLES WITH READ LOCK;"
+      
+      res = conn.query "SHOW MASTER STATUS"
+      logfile, position = res.fetch_row[0..1]
+# puts "Snapshot occuring at: log: #{logfile}, #{position}"
+      vols.each_pair do |mount, ebs_info|
+        begin
+          `sudo xfs_freeze -f #{mount}`
+          output = ec2.create_snapshot(:volume_id => ebs_info['volume_id'])
+          snap_id = output['CreateSnapshotResponse']['snapshotId'] rescue nil
+          snap_id ||= output['snapshotId'] rescue nil #this is for the old version of the amazon-ec2
+          if snap_id.nil? || snap_id.empty?
+            puts "Snapshot for #{ebs_info['volume_id']} FAILED"
+            exit
+          end
+          vol_id  = ebs_info['volume_id']
+       ensure
+          `sudo xfs_freeze -u #{mount}`
+        end
+      end
+    ensure
+      conn.query <<-SQL
+      UNLOCK TABLES;
+      SQL
+    end
+  end
+else
+  #not persisted, so lets push the binary log files to s3
+  # include the hostname in the bucket name so test instances don't accidentally clobber real backups
+  bucket = ARGV.flags.bucket
+  dir = ARGV.flags.dir || "database"
+  @s3 = Ec2onrails::S3Helper.new(bucket, dir)
+  @temp_dir = "/mnt/tmp/ec2onrails-backup-#{@s3.bucket_name}-#{dir.gsub(/\//, "-")}"
+  if File.exists?(@temp_dir)
+    puts "Temp dir exists (#{@temp_dir}), aborting. Is another backup process running?"
+    exit
+  end
+
+  begin
+    FileUtils.mkdir_p @temp_dir
+    if ARGV.flags.incremental
+      # Incremental backup
+      @mysql.execute_sql "flush logs"
+      logs = Dir.glob("/mnt/log/mysql/mysql-bin.[0-9]*").sort
+      logs_to_archive = logs[0..-2] # all logs except the last
+      logs_to_archive.each {|log| @s3.store_file log}
+      @mysql.execute_sql "purge master logs to '#{File.basename(logs[-1])}'"
+    else
+      # Full backup
+      file = "#{@temp_dir}/dump.sql.gz"
+      @mysql.dump(file, ARGV.flags.reset)
+      @s3.store_file file
+      @s3.delete_files("mysql-bin")
+    end
+  ensure
+    FileUtils.rm_rf(@temp_dir)
+  end
+
+end
+

data/server/files/usr/local/ec2onrails/bin/backup_dir

@@ -0,0 +1,89 @@
+#!/usr/bin/ruby
+      
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+  
+require "rubygems"
+require "optiflag"
+require "fileutils"
+require 'EC2'
+require "#{File.dirname(__FILE__)}/../lib/mysql_helper"
+require "#{File.dirname(__FILE__)}/../lib/s3_helper"
+require "#{File.dirname(__FILE__)}/../lib/aws_helper"
+require "#{File.dirname(__FILE__)}/../lib/roles_helper"
+
+require "#{File.dirname(__FILE__)}/../lib/utils"
+
+# Only run if this instance is the db_pimrary
+# The original code would run on any instance that had /etc/init.d/mysql
+# Which was pretty much all instances no matter what role
+include Ec2onrails::RolesHelper
+
+
+module CommandLineArgs extend OptiFlagSet
+  curr_env = Ec2onrails::Utils.rails_env
+  default_bucket = "#{curr_env}_backup"
+
+  flag "dir" do
+    description "the directory that will be tarred and compressed and put on S3 with the name DIR_#{Ec2onrails::Utils.hostname}_TIMESTAMP.tgz"
+  end
+  
+  optional_flag "role" do
+    description "The role of this server, as defined by capistrano. ex. 'db', or 'app'  If not used, will be applied to all roles"
+  end
+
+  optional_flag "only_env" do
+    description "Only apply the script if it is running within this environment"
+  end
+  
+  optional_flag "bucket" do
+    description "The s3 bucket you would like to save this backup to.  Will default to #{default_bucket}"
+  end
+
+  optional_switch_flag "v" do
+    description "let you know if the script stopped because it was running in either a different role or environment than the one specified"
+  end
+  
+  and_process!
+end
+curr_env = Ec2onrails::Utils.rails_env
+default_bucket = "#{curr_env}_backup"
+
+verbose = ARGV.flags.v
+dir = ARGV.flags.dir
+bucket = ARGV.flags.bucket || default_bucket
+curr_env = Ec2onrails::Utils.rails_env
+default_bucket = "#{curr_env}_backup"
+
+if ARGV.flags.role && !in_role?(ARGV.flags.role.sub(/^:/, '').to_sym)
+  puts "This script is not being run because the server is not running under the #{role} role" if verbose
+  exit
+end
+
+if ARGV.flags.only_env && ARGV.flags.only_env.strip.downcase != curr_env.strip.downcase
+  puts "This script is not being run because the server is not running under the #{curr_env} environment" if verbose
+  exit
+end
+
+if !dir || File.exists?(dir)
+  puts "The directory '#{dir}' does not exist.  Please enter a valid, full path to a directory you would like backed up" if verbose
+end
+
+
+@s3 = Ec2onrails::S3Helper.new(bucket, dir)
+@s3.store_dir(dir, :compress => true)

data/server/files/usr/local/ec2onrails/bin/ec2_meta_data

@@ -0,0 +1,80 @@
+#!/usr/bin/ruby
+
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+#    This file helps in setting up and retrieving ec2 meta-data
+#    If a key is passed in, it will only retrieve that key
+#    if not, then it will return all of the meta-data formatted in yaml
+#
+
+require "rubygems"
+require "optiflag"
+require 'yaml'
+require "#{File.dirname(__FILE__)}/../lib/roles_helper"
+include Ec2onrails::RolesHelper
+
+CURL_OPTS  = "-s -S -f -L --retry 7"
+META_URL   = "http://169.254.169.254/latest/meta-data"
+
+
+module CommandLineArgs extend OptiFlagSet
+  optional_flag "key" do
+    description "The ec2 meta-data value you would like to look up"
+  end
+  and_process!
+end
+
+
+def process_files(files, root_url="")
+  output = {}
+  files.split.each do |file|
+    if file =~ /\/$/
+      output.merge! process_files(`curl #{CURL_OPTS} #{META_URL}/#{file}`, "#{file}") 
+    else      
+      if file =~ /=/
+        key, data = file.split('=')        
+      else
+        url = "#{META_URL}/#{root_url}/#{file}"
+        data = `curl #{CURL_OPTS} #{url}`
+        raise "Failed to fetch entry #{file}: code #{$?.exitstatus} -- #{data}" unless $?.success?
+      end
+      if root_url.nil? || root_url.strip.length == 0
+        output[file] = data
+      else
+        output[root_url] ||= {}
+        output[root_url][file] = data
+      end
+    end
+  end  
+  output
+end
+
+
+
+if ARGV.flags.key
+  puts get_metadata(ARGV.flags.key) 
+else
+
+  files = `curl #{CURL_OPTS} #{META_URL}/`
+  raise "Failed to fetch directory: code #{$?.exitstatus} -- #{files}" unless $?.success?
+  val = process_files(files)
+  puts val.to_yaml
+end
+

data/server/files/usr/local/ec2onrails/bin/exec_runner

@@ -0,0 +1,76 @@
+#!/usr/bin/ruby
+
+#    This file is part of EC2 on Rails.
+#    http://rubyforge.org/projects/ec2onrails/
+#
+#    Copyright 2007 Paul Dowman, http://pauldowman.com/
+#
+#    EC2 on Rails is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    EC2 on Rails is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#    This script allows you to ONLY execute within a shell IF the role of
+#    the server (as defined by capistrano) matches the role specified.
+#    ex.
+#      * exec_runner -role :db -exec 'echo "hello world"'
+#        # will only run if the server is in the db role
+#      * exec_runner -role :web -exec /some/app/specific/script/to/call
+#
+
+require "rubygems"
+require "optiflag"
+
+require "fileutils"
+require "#{File.dirname(__FILE__)}/../lib/roles_helper"
+require "#{File.dirname(__FILE__)}/../lib/utils"
+
+include Ec2onrails::RolesHelper
+
+module CommandLineArgs extend OptiFlagSet
+  optional_flag "role" do
+    description "The role of this server, as defined by capistrano. ex. 'db', or 'app'  If not used, will be applied to all roles"
+  end
+  
+  optional_flag "only_env" do
+    description "Only apply the script if it is running within this environment"
+  end
+
+  flag "exec" do
+    description "what to run if the role of the server matches the -role passed in"
+  end
+  
+  optional_switch_flag "v" do
+    description "let you know if the script stopped because it was running in either a different role or environment than the one specified"
+  end
+  
+  and_process!
+end
+
+#strip out the ':', in case the user enters ':db', or ':web'
+if ARGV.flags.role && !in_role?(ARGV.flags.role.sub(/^:/, '').to_sym)
+  puts "This script is not being run because the server is not running under the #{role} role" if ARGV.flags.v
+  exit
+end
+
+curr_env = Ec2onrails::Utils.rails_env
+if ARGV.flags.only_env && ARGV.flags.only_env.strip.downcase != curr_env.strip.downcase
+  puts "This script is not being run because the server is not running under the #{curr_env} environment" if ARGV.flags.v
+  exit
+end
+
+
+# set the default to the current directory; makes it easier to 
+Dir.chdir('/mnt/app/current')
+
+ENV['RAILS_ENV'] = curr_env
+run ARGV.flags.exec
+