sr-scripts 0.0.2

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in sr-scripts.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/bin/sr-backup-mysql

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'fog'
+require 'mysql'
+
+connection = Fog::Compute.new(:provider => "AWS", :region => "us-west-1")
+
+current_instance_id = `curl -s http://169.254.169.254/latest/meta-data/instance-id`
+
+server = connection.servers.get(current_instance_id)
+
+disks = server.tags["mysql_disks"].split(":")
+
+volumes = []
+server.block_device_mapping.each do |b|
+	if disks.include? b["deviceName"]
+		volumes.push b["volumeId"]
+	end
+end
+
+if volumes.length == 0 
+	p "No Volumes To Snapshot" 
+	exit
+end
+
+#check here that there aren't current pending snapshots
+pending = connection.snapshots.find_all { |s| s.state == "pending" }.find_all { |s| volumes.include? s.volume_id }.length != 0
+if pending
+	p "Exiting: Pending Snapshots Exist"
+	exit
+end
+#p "./ec2-consistent-snapshot-rb -f conf.yml #{volumes.join(' ')}"
+`sr-ec2-consistent-snapshot -f /etc/sr-ec2-consistent-snapshot.yml #{volumes.join(' ')}`

data/bin/sr-ec2-consistent-snapshot

@@ -0,0 +1,152 @@
+#!/usr/bin/env ruby
+
+# This script was largely ported from ec2-consistent-snapshot written
+# by Eric Hammond: http://alestic.com/2009/09/ec2-consistent-snapshot
+# A bunch of his features aren't ported over yet... just the stuff I needed.
+
+require 'rubygems'
+require 'optparse'
+require 'fog'
+
+$opts = {
+  :aws_access_key => ENV["AWS_ACCESS_KEY_ID"],
+  :aws_secret_access_key => ENV["AWS_SECRET_ACCESS_KEY"],
+  :aws_region => 'us-east-1',
+  
+  :description => '',
+  :xfs_filesystem => nil,
+  
+  :mysql => false,
+  :mysql_username => 'root',
+  :mysql_password => nil,
+  :mysql_host => '127.0.0.1',
+  :mysql_master_status_file => nil,
+  :mysql_slave_status_file => nil,
+}
+
+OptionParser.new do |o|
+  o.on("--aws-access-key ACCESS_KEY", "AWS Access Key") {|v| $opts[:aws_access_key] = v }
+  o.on("--aws-secret-access-key SECRET_KEY", "AWS Secret Access Key") {|v| $opts[:aws_secret_access_key] = v }
+  o.on("--aws-region REGION", "AWS Region") {|v| $opts[:aws_region] = v }
+  o.on("--description STRING", "The description for the snapshot") {|v| $opts[:description] = v }
+  o.on("--xfs-filesystem MOUNTPOINT", "Filesystem to be frozen during snapshot") {|v| $opts[:xfs_filesystem] = v }
+  o.on("--mysql", "Indicates that the volume has mysql") {|v| $opts[:mysql] = v }
+  o.on("--mysql-username USERNAME", "MySQL user (default: root)") {|v| $opts[:mysql_username] = v }
+  o.on("--mysql-password PASSWORD", "MySQL password (default: none)") {|v| $opts[:mysql_password] = v }
+  o.on("--mysql-host HOST", "MySQL host (default: 127.0.0.1)") {|v| $opts[:mysql_host] = v }
+  o.on("--mysql-master-status-file FILENAME", "File to store in snapshot with master status") {|v| $opts[:mysql_master_status_file] = v }
+  o.on("--mysql-slave-status-file FILENAME", "File to store in snapshot with slave status") {|v| $opts[:mysql_slave_status_file] = v }
+  o.on("-f", "--configfile PATH", String, "Set config file") do |path|
+    $opts.merge!(Hash[YAML::load(open(path)).map { |k, v| [k.to_sym, v] }])
+    #p Hash[YAML::load(open(path)).map { |k, v| [k.to_sym, v] }]
+  end
+end.parse!
+
+p $opts
+
+if $opts[:aws_access_key].nil? || $opts[:aws_secret_access_key].nil?
+  puts "You must specify your Amazon credentials via --aws-access-key and --aws-secret_access-key"
+  exit 1
+end
+
+if ARGV.empty?
+  puts "You must provide at least one volume id to snapshot"
+  exit 1
+end
+volume_ids = ARGV
+
+def mysql_locked(&block)
+  mysql = nil
+  
+  if $opts[:mysql]
+    require 'mysql'
+    mysql = Mysql::new($opts[:mysql_host], $opts[:mysql_username], $opts[:mysql_password], nil, nil, '/mnt/mysql/mysql/mysql.sock')
+    mysql.query("SET SQL_LOG_BIN=0")
+    mysql.query("FLUSH LOCAL TABLES")
+    mysql.query("FLUSH LOCAL TABLES WITH READ LOCK")
+    
+    def query_result_string(mysql, query)
+      result = mysql.query(query)
+      string = ""
+      if result.num_rows() > 0
+        result.fetch_row.each_with_index do |value, i|
+          string << "#{result.fetch_field_direct(i).name}: #{value}\n"
+        end
+      end
+      string
+    end
+    
+    if $opts[:mysql_slave_status_file]
+      File.open($opts[:mysql_slave_status_file], "w").puts query_result_string(mysql, "SHOW SLAVE STATUS")
+    end
+
+		rs = mysql.query("SHOW MASTER STATUS")
+		if rs.num_rows() == 1 
+			row = rs.fetch_hash
+			master_log_file = row["File"]
+			master_log_pos = row["Position"]		
+			master_info = Hash.new
+			master_info["MASTER_LOG_FILE"] = master_log_file
+			master_info["MASTER_LOG_POS"] = master_log_pos
+		end
+			puts master_log_file 
+    
+    if $opts[:mysql_master_status_file]
+      File.open($opts[:mysql_master_status_file], "w").puts query_result_string(mysql, "SHOW MASTER STATUS")
+    end
+    
+    mysql.query("SET SQL_LOG_BIN=1")
+  end
+  
+  begin
+    yield master_info
+  ensure
+    mysql.real_query("UNLOCK TABLES") if mysql
+  end
+end
+
+def xfs_frozen(&block)
+  system('xfs_freeze', '-f', $opts[:xfs_filesystem]) if $opts[:xfs_filesystem]
+  
+  begin
+    yield
+  ensure
+    system('xfs_freeze', '-u', $opts[:xfs_filesystem]) if $opts[:xfs_filesystem]
+  end
+end
+
+snapshots = []
+master_info = nil
+
+connection = Fog::Compute.new(:provider => "AWS", :aws_access_key_id => $opts[:aws_access_key], :aws_secret_access_key => $opts[:aws_secret_access_key], :region => $opts[:aws_region])
+sdb = Fog::AWS::SimpleDB.new(:aws_access_key_id => $opts[:aws_access_key], :aws_secret_access_key => $opts[:aws_secret_access_key])
+begin
+  mysql_locked() do |info|
+    master_info = info
+    xfs_frozen() do
+      volume_ids.each do |volume_id|
+				snapshot = connection.snapshots.create(:volume_id => volume_id, :description => $opts[:description])
+	
+	snapshots.push(snapshot.id)
+        puts "#{volume_id}"
+      end
+    end
+  end
+end
+
+random_id = rand(36**8).to_s(36)
+attributes = Hash.new
+attributes["instance_id"] = `curl http://169.254.169.254/latest/meta-data/instance-id`
+attributes["snapshots"] = snapshots.join(",")
+attributes["timestamp"] = Time.now.to_i
+attributes["master_log_file"] = master_info["MASTER_LOG_FILE"]
+attributes["master_log_pos"] = master_info["MASTER_LOG_POS"]
+sdb.put_attributes("db_recovery_info", random_id, attributes)
+
+snapshots.each do |s|
+	attributes.each do |key, value|
+		connection.tags.create(:resource_id => s, :key => key, :value => value)
+	end
+end
+
+p master_info

data/bin/sr-start-slave

@@ -0,0 +1,117 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'fog'
+require 'mysql'
+require 'ostruct'
+
+connection = Fog::Compute.new(:provider => "AWS", :region => "us-west-1")
+sdb = Fog::AWS::SimpleDB.new()
+
+# somewhere up here check to see if the devices are already attached
+
+p connection.snapshots.all.first
+
+if ARGV.length != 1
+	p "Must specify an instance id"
+	exit
+end
+
+instance_id = ARGV[0] 
+
+records = sdb.select("SELECT * FROM db_recovery_info WHERE instance_id = '#{instance_id}' AND timestamp > '1' ORDER BY timestamp DESC LIMIT 5").body["Items"]
+
+p records
+
+rows = []
+records.each_value do |r|
+	r.each do |k,v|
+		r[k] = v[0]
+	end
+	data = OpenStruct.new(r)
+	rows.push data
+end
+
+rows.sort! { |a,b| b.timestamp <=> a.timestamp }
+
+latest_snapshots = nil
+rows.each do |r|
+	if latest_snapshots != nil
+		break
+	end
+	pending = false
+	r.snapshots.split(",").each do |snap|
+		if connection.snapshots.get(snap).state == "pending"
+			pending = true
+		end
+			
+	end
+	if pending == false
+		latest_snapshots = r.snapshots.split(",")
+	end
+	
+end
+
+#latest_snapshots now equals the snapshot ids
+p latest_snapshots
+
+def get_latest_snapshot(snapshots, instance_id)
+        filtered_snap = snapshots.find_all { |s| s.tags["instance_id"] == instance_id }
+        latest_snap = filtered_snap.sort { |a,b| b.created_at <=> a.created_at }.first unless filtered_snap.length == 0
+        return latest_snap
+end
+
+snapshot = get_latest_snapshot(connection.snapshots.all, instance_id)
+
+p snapshot
+
+# USE THE TAGS in snapshot to call CHANGE MASTER TO
+
+current_master = connection.servers.get(instance_id)
+
+# VALIDATE THAT NUMBER OF SNAPSHOTS EQUALS NUMBER OF MYSQL DISKS
+
+master_disks = current_master.tags["mysql_disks"].split(":")
+
+if latest_snapshots.length != master_disks.length
+	p "EXITING: Number of Snapshots != Number of Disks"
+	exit
+end
+
+p current_master
+# need to find out the current node that this is running on
+#
+#
+current_instance_id = `curl http://169.254.169.254/latest/meta-data/instance-id`
+current_instance = connection.servers.get(current_instance_id)
+
+latest_snapshots.each_index |index| do
+	snapshot_id = latest_snapshots[index]
+	current_disk = master_disks[index]
+	
+	snapshot = connection.snapshots.get(snapshot_id)
+
+	vol = connection.volumes.new(:snapshot_id => snapshot_id, :availability_zone => current_instance.availability_zone, :size => snapshot.volume_size)
+	vol.device = current_disk 
+	vol.server = current_instance
+	vol.save
+	p vol
+
+	until vol.state == "in-use" do
+		sleep 1
+		vol.reload
+	end
+end
+
+exit
+
+sleep 15
+
+# NEED TO FIND OUT IF NUMBER OF DISKS IS GREATER THAN 1, IF SO, THEN use mdadm to create an array and mount, if not, then just mount
+
+`mkdir /mnt/mysql`
+`mount -t xfs /dev/sdk /mnt/mysql`
+`service mysql start`
+
+#`mysql -e "CHANGE MASTER TO MASTER_HOST='#{current_master.private_ip_address}', MASTER_LOG_FILE='binarylogs.006693', MASTER_LOG_POS=785268118;"`
+# start mysql, connect to mysql and issue change master and start slave commands

data/lib/sr-scripts/version.rb

@@ -0,0 +1,5 @@
+module Sr
+  module Scripts
+    VERSION = "0.0.2"
+  end
+end

data/lib/sr-scripts.rb

@@ -0,0 +1,5 @@
+module Sr
+  module Scripts
+    # Your code goes here...
+  end
+end

data/sr-scripts.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "sr-scripts/version"
+
+Gem::Specification.new do |s|
+  s.name        = "sr-scripts"
+  s.version     = Sr::Scripts::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Davy Campano"]
+  s.email       = ["dcampano@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Admin scripts}
+  s.description = %q{Admin scripts}
+
+  s.rubyforge_project = "sr-scripts"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification 
+name: sr-scripts
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
+platform: ruby
+authors: 
+- Davy Campano
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-02-18 00:00:00 -06:00
+default_executable: 
+dependencies: []
+
+description: Admin scripts
+email: 
+- dcampano@gmail.com
+executables: 
+- sr-backup-mysql
+- sr-ec2-consistent-snapshot
+- sr-start-slave
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- Rakefile
+- bin/sr-backup-mysql
+- bin/sr-ec2-consistent-snapshot
+- bin/sr-start-slave
+- lib/sr-scripts.rb
+- lib/sr-scripts/version.rb
+- sr-scripts.gemspec
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: sr-scripts
+rubygems_version: 1.5.2
+signing_key: 
+specification_version: 3
+summary: Admin scripts
+test_files: []
+