rest_connection 0.0.1

data/README

@@ -0,0 +1,7 @@
+RestConnection is a library designed to facilitate Restful connections to APIs using Basic auth and more.  However!  If you've done any API work you know that every API is different.  The goal of RestConnection is to be as lean as possible and a EXAMPLE of how to use Net::HTTP in a restful way.  Right now RestConnection uses JSON and Basic Auth, this is easy to change or expand on and it's written to give more control to the user of the library (You).
+
+My hope is to include 'helper resources' for common API endpoints while maintaining a somewhat universal 'Connection' class that can be easily understood and modified to suit any Restful API needs.  These resources will have a similar usage to ActiveResource. 
+
+The first API we will be supporting is the RightScale API.  
+
+I welcome any contributions or suggestions.  Thanks!

data/Rakefile

@@ -0,0 +1,14 @@
+require 'rubygems'
+require 'jeweler'
+Jeweler::Tasks.new do |gemspec|
+  gemspec.name = "rest_connection"
+  gemspec.summary = "lib for restful connections to the rightscale api"
+  gemspec.description = "provides rest_connection"
+  gemspec.email = "jeremy@rubyonlinux.org"
+  gemspec.homepage = "http://github.com/jeremyd/rest_connection"
+  gemspec.authors = ["Jeremy Deininger"]
+  gemspec.add_dependency('mechanize', '>= 0.9.3')
+  gemspec.add_dependency('activesupport')
+end
+Jeweler::GemcutterTasks.new
+

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/config/rest_api_config.yaml.sample

@@ -0,0 +1,7 @@
+--- 
+:pass: myUltraSecurePasswordz
+:user: myUserEmail@MyMailDomain.com
+:api_url: https://my.rightscale.com/api/acct/00000000
+:ssh_key: ~/.ssh/my-server-key
+:common_headers: 
+  X_API_VERSION: "1.0"

data/examples/console.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'rest_connection'
+require 'ruby-debug'
+
+puts "entering debug session"
+debugger
+puts "console session commplete"
+
+
+

data/examples/dev_setup.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/ruby
+require 'rubygems'
+require 'trollop'
+require 'rest_connection'
+require 'net/ssh'
+
+opts = Trollop::options do
+  opt :deployment, "deployment nickname", :type => :string, :required => true
+  opt :only, "regex string matching the nickname of the servers you want to relaunch. This excludes servers that do not match\nExample --only ubuntu", :type => :string, :required => false
+  opt :growl, "use growl notification (string) when servers are all operational, requires ruby-growl gem", :type => :string, :required => false
+end
+
+# find all servers in the deployment (the fast way)
+deployment = Deployment.find_by_nickname_speed(opts[:deployment]).first
+servers = deployment.servers_no_reload
+servers = servers.select { |s| s.nickname =~ /#{opts[:only]}/ } if opts[:only]
+
+servers.each do |s|
+  s.wait_for_operational_with_dns
+  
+  Net::SSH.start(s.dns_name, 'root', :keys => ['~/.ssh/publish-test']) do |ssh|
+    puts "setting up devmode: dropbox::install && devmode::setup_cookbooks"
+    puts ssh.exec!("rs_run_recipe -n 'dropbox::install'")
+    puts ssh.exec!("rs_run_recipe -n 'devmode::setup_cookbooks'")
+  end
+end

data/examples/mysql_dev_rollback.rb

@@ -0,0 +1,29 @@
+#!/usr/bin/ruby
+require 'rubygems'
+require 'trollop'
+require 'rest_connection'
+require 'net/ssh'
+
+opts = Trollop::options do
+  opt :deployment, "deployment nickname", :type => :string, :required => true
+  opt :only, "regex string matching the nickname of the servers you want to relaunch. This excludes servers that do not match\nExample --only ubuntu", :type => :string, :required => false
+  opt :hard, "hard reset, uninstalls right_resources premium gem (for code update), and restarts the instance agent", :type => :bool, :required => false
+end
+
+# find all servers in the deployment (the fast way)
+deployment = Deployment.find_by_nickname_speed(opts[:deployment]).first
+servers = deployment.servers_no_reload
+servers = servers.select { |s| s.nickname =~ /#{opts[:only]}/ } if opts[:only]
+
+servers.each do |s|
+  s.wait_for_operational_with_dns
+  if opts[:hard]
+    s.spot_check("gem uninstall right_resources_premium") do |result|
+      puts result
+    end
+    s.spot_check("monit restart instance") do |result|
+      puts result
+    end
+  end
+  s.run_recipe("database_test::dev_pristine_restore")
+end

data/examples/relaunch_deployment.rb

@@ -0,0 +1,45 @@
+#!/usr/bin/ruby
+require 'rubygems'
+require 'trollop'
+require 'rest_connection'
+
+opts = Trollop::options do
+  opt :deployment, "deployment nickname", :type => :string, :required => false
+  opt :only, "regex string matching the nickname of the servers you want to relaunch. This excludes servers that do not match\nExample --only ubuntu", :type => :string, :required => false
+  opt :growl, "use growl notification (string) when servers are all operational, requires ruby-growl gem", :type => :string, :required => false
+  opt :id, "deployment id", :type => :string, :required => false
+end
+
+# find all servers in the deployment (the fast way)
+if opts[:id]
+  deployment = Deployment.find_by_id(opts[:id])
+else
+  deployment = Deployment.find_by_nickname_speed(opts[:deployment]).first
+end
+servers = deployment.servers_no_reload
+servers = servers.select { |s| s.nickname =~ /#{opts[:only]}/ } if opts[:only]
+servers.each do |s|
+  # send stop
+  s.relaunch
+end
+
+# wait for termination
+#servers.each do |s|
+#  s.wait_for_state('stopped')
+#end
+
+# relaunch
+#servers.each do |s|
+#  s.start
+#end
+
+#if opts[:growl]
+#  require 'ruby-growl'
+#  servers.each do |s|
+#    s.wait_for_state('booting')
+#  end
+#  g = Growl.new "localhost", "ruby-growl",
+#              ["ruby-growl Notification"]
+#g.notify "ruby-growl Notification", "It Came From Ruby-Growl",
+#         "Greetings!"
+#end

data/examples/restart_instance_agent.rb

@@ -0,0 +1,29 @@
+#!/usr/bin/ruby
+require 'rubygems'
+require 'trollop'
+require 'rest_connection'
+require 'net/ssh'
+
+opts = Trollop::options do
+  opt :deployment, "deployment nickname", :type => :string, :required => true
+  opt :only, "regex string matching the nickname of the servers you want to relaunch. This excludes servers that do not match\nExample --only ubuntu", :type => :string, :required => false
+  opt :hard, "hard reset, uninstalls right_resources premium gem (for code update), and restarts the instance agent", :type => :bool, :required => false
+end
+
+# find all servers in the deployment (the fast way)
+deployment = Deployment.find_by_nickname_speed(opts[:deployment]).first
+servers = deployment.servers_no_reload
+servers = servers.select { |s| s.nickname =~ /#{opts[:only]}/ } if opts[:only]
+
+servers.each do |s|
+  s.wait_for_operational_with_dns
+  if opts[:hard]
+    s.spot_check("gem uninstall right_resources_premium") do |result|
+      puts result
+    end
+    s.spot_check("monit restart instance") do |result|
+      puts result
+    end
+  end
+  s.run_recipe("database_test::dev_pristine_restore")
+end

data/examples/right_scale_ec2_instances_api_test.rb

@@ -0,0 +1,51 @@
+require 'rubygems'
+require 'rest_connection'
+require 'spec'
+require '/var/spool/ec2/user-data'
+
+describe Instance, "run this from an ec2_instance" do
+  before(:all) do
+    @this_instance = Instance.new
+    @this_instance.connection.settings[:api_url] = ENV['RS_API_URL']
+    @this_instance.connection.settings[:common_headers] = { 'X_API_VERSION' => '1.0' }
+    # when tests fail, we need this to ensure a fresh run
+    x = @this_instance.detach_ebs_volume(:device => '/dev/sdp')
+    sleep 10 if x
+  end
+  it "should create,attach,detach,delete an ebs_volume" do
+    sleep 2
+    result = @this_instance.create_ebs_volume({ 
+                :nickname => "ebs_test_candelete#{rand(1000)}",
+                :description => "created by ebs integration",
+                :size => "1" })
+    vol_aws_id = result['aws_id']
+    vol_aws_id.should_not == nil
+    @this_instance.attach_ebs_volume(:aws_id => result['aws_id'], :device => "/dev/sdp")
+    sleep 10
+    @this_instance.create_ebs_snapshot(:aws_id => vol_aws_id)
+    sleep 5
+    @this_instance.detach_ebs_volume(:device => '/dev/sdp')
+    sleep 10
+    @this_instance.delete_ebs_volume(:aws_id => vol_aws_id)
+  end
+
+end
+
+=begin
+#ROUTES REFERENCE GUIDE
+ map.api_inst_resources 'ec2_instances', :member => { :find_latest_ebs_snapshot => :get,
+                                                         :find_ebs_snapshots => :get, 
+                                                         :create_ebs_volume_from_snap => :post, 
+                                                         :create_ebs_volume => :post,
+                                                         :delete_ebs_volume => :delete, 
+                                                         :detach_ebs_volume => :put, 
+                                                         :attach_ebs_volume => :put,
+                                                         :cleanup_ebs_snapshots => :put,
+                                                         :create_ebs_snapshot => :post,
+                                                         :update_ebs_snapshot => :put,
+                                                         :create_ebs_backup => :post,
+                                                         :find_latest_ebs_backup => :get,
+                                                         :cleanup_ebs_backups => :put,
+                                                         :set_custom_lodgement => :put  # should we be using :post or :put ???
+                                                       }
+=end

data/examples/run_ebs_sequence.rb

@@ -0,0 +1,95 @@
+#!/usr/bin/ruby
+require 'rubygems'
+require 'trollop'
+require 'rest_connection'
+#
+# EBS regression test
+# 
+# prerequirements: run the macro for EBS stripe deployment
+# enter the deployment name below
+#
+
+opts = Trollop::options do
+  opt :deployment, "deployment nickname", :type => :string, :required => true
+end
+
+#ebs_deployment = Deployment.find_by_nickname_speed("Regression Test - MySQL -STRIPE").first
+ebs_deployment = Deployment.find_by_nickname_speed(opts[:deployment]).first
+
+# select servers by nickname!
+mysql_s3 = ebs_deployment.servers.detect { |s| s.nickname =~ /MySQL S3 US db1/i }
+mysql_db1 = ebs_deployment.servers.detect { |s| s.nickname =~ /MySQL EBS db1/i }
+mysql_db2 = ebs_deployment.servers.detect { |s| s.nickname =~ /MySQL EBS Stripe db1/i }
+mysql_db3 = ebs_deployment.servers.detect { |s| s.nickname =~ /MySQL EBS Stripe db2/i }
+
+# we must reload, so the server can populate it's state.  we took a shortcut
+# by grabbing the servers from the deployment's json (the rightscale default response)
+mysql_s3.reload
+mysql_db1.reload
+mysql_db2.reload
+mysql_db3.reload
+
+# tweak: (this is closer to an actual user story) setup mysql_db1 for regular EBS
+# the rest of the deployment is normally set to text:2
+# mysql_db1 is [rev 13] so it needs the old input name
+
+#mysql_db1.set_input("OPT_EBS_STRIPE_COUNT", "ignore:$ignore")
+#mysql_db1.set_input("DB_EBS_PREFIX", "text:shouldnotmatter-candelete652")
+#mysql_db1.set_input("DB_LINEAGE_NAME", "text:shouldnotmatter-candelete652")
+#mysql_db2.set_input("DB_LINEAGE_NAME", "text:shouldnotmatter-candelete652")
+#mysql_db3.set_input("DB_LINEAGE_NAME", "text:shouldnotmatter-candelete652")
+
+# launch all
+mysql_s3.start
+mysql_db1.start
+mysql_db2.start
+mysql_db3.start
+
+# wait for operational
+mysql_s3.wait_for_state('operational')
+mysql_db1.wait_for_state('operational')
+mysql_db2.wait_for_state('operational')
+mysql_db3.wait_for_state('operational')
+
+sleep 10 #make sure they are operational
+
+# we need better api support, but for now we can get our RightScripts from info.yml on an instance (or in the current dir)
+template_scripts = RightScript.from_instance_info
+
+# scripts from EBS [rev13]
+slave_init_non_ebs_rev4 = RightScript.new('href' => 'right_scripts/45320')
+slave_init_non_ebs_v1_rev6 = RightScript.new('href' => 'right_scripts/41484')
+restore_rev4 = RightScript.new('href' => 'right_scripts/45319')
+
+# see how easy this is to cherry pick the scripts by name? this means no resetting of ids across different templates
+restore_script = template_scripts.detect { |script| script.name =~ /restore/i }
+slave_init_non_ebs = template_scripts.detect { |script| script.name =~ /DB EBS slave init from non-EBS master v1/ }
+slave_init_non_stripe = template_scripts.detect { |script| script.name =~ /DB EBS stripe slave init from EBS non-stripe master/i }
+promote = template_scripts.detect { |script| script.name =~ /promote/i }
+slave_init = template_scripts.detect { |script| script.name == "DB EBS slave init (Stripe alpha)" }
+backup = template_scripts.detect { |script| script.name =~ /DB EBS backup/i }
+
+# breathing room is needed below *only to let the servers
+# settle after fail-over operations. wait_for_completed returns
+# when the right_script status is 'success' or 'fail'
+
+status = mysql_db1.run_script(slave_init_non_ebs_v1_rev6)
+status = mysql_db1.run_script(slave_init_non_ebs_rev4)
+status.wait_for_completed(mysql_db1.audit_link)
+sleep 220 #breathing room
+
+status = mysql_db1.run_script(promote)
+status.wait_for_completed(mysql_db1.audit_link)
+sleep 220 #breathing room
+
+status = mysql_db2.run_script(slave_init_non_stripe)
+status.wait_for_completed(mysql_db2.audit_link)
+sleep 220 # breathing room
+
+status = mysql_db2.run_script(promote)
+status.wait_for_completed(mysql_db2.audit_link)
+sleep 220 #breathing room
+
+status = mysql_db3.run_script(slave_init)
+status.wait_for_completed(mysql_db3.audit_link)
+puts "done!"

data/examples/run_ebs_terminate.rb

@@ -0,0 +1,40 @@
+#!/usr/bin/ruby
+require 'rubygems'
+require 'trollop'
+require File.join(File.dirname(__FILE__), '..', 'lib', 'rightscale_api_resources')
+
+#
+# EBS regression test
+# 
+# prerequirements: run the macro for EBS stripe deployment
+# enter the deployment name below
+#
+
+opts = Trollop::options do
+  opt :deployment, "deployment nickname", :type => :string, :required => true
+end
+
+ebs_deployment = Deployment.find_by_nickname_speed(opts[:deployment]).first
+# select servers by nickname!
+mysql_s3 = ebs_deployment.servers.detect { |s| s.nickname =~ /MySQL S3 US db1/i }
+mysql_db1 = ebs_deployment.servers.detect { |s| s.nickname =~ /MySQL EBS db1/i }
+mysql_db2 = ebs_deployment.servers.detect { |s| s.nickname =~ /MySQL EBS Stripe db1/i }
+mysql_db3 = ebs_deployment.servers.detect { |s| s.nickname =~ /MySQL EBS Stripe db2/i }
+
+# we must reload, so the server can populate it's state
+mysql_s3.reload
+mysql_db1.reload
+mysql_db2.reload
+mysql_db3.reload
+
+# stop all
+mysql_s3.stop
+mysql_db1.stop
+mysql_db2.stop
+mysql_db3.stop
+
+# wait for stopped
+mysql_s3.wait_for_state('stopped')
+mysql_db1.wait_for_state('stopped')
+mysql_db2.wait_for_state('stopped')
+mysql_db3.wait_for_state('stopped')

data/examples/run_mysql_chef_sequence.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/ruby
+require 'rubygems'
+require 'trollop'
+require 'rest_connection'
+require 'net/ssh'
+
+opts = Trollop::options do
+  opt :deployment, "deployment nickname", :type => :string, :required => false
+  opt :only, "regex string matching the nickname of the servers you want to relaunch. This excludes servers that do not match\nExample --only ubuntu", :type => :string, :required => false
+  opt :id, "deployment id", :type => :string, :required => false
+end
+
+# find all servers in the deployment (the fast way)
+if opts[:id]
+  deployment = Deployment.find_by_id(opts[:id])
+else
+  deployment = Deployment.find_by_nickname_speed(opts[:deployment]).first
+end
+servers = deployment.servers_no_reload
+servers = servers.select { |s| s.nickname =~ /#{opts[:only]}/ } if opts[:only]
+
+raise "need at least 2 servers to start, only have: #{servers.size}" if servers.size < 2
+
+# wait for servers to be ready
+servers.each do |s|
+  s.start
+  while(1)
+    puts "waiting for dns-name for #{s.nickname}"
+    break if s['dns-name'] && !s['dns-name'].empty?
+    s.reload
+    sleep 2
+  end
+  puts "DNS: #{s['dns-name']}"
+
+  s.wait_for_state('operational')
+end  
+
+servers[0].run_recipe("db_mysql::do_restore_and_become_master")
+
+sleep(10)
+
+servers[1].run_recipe("db_mysql::do_init_slave")
+servers[1].run_recipe("db_mysql::do_promote_to_master")

data/examples/run_php_chef_sequence.rb

@@ -0,0 +1,109 @@
+#!/usr/bin/ruby
+require 'rubygems'
+require 'trollop'
+require 'rest_connection'
+require 'net/ssh'
+
+def run_recipe(recipe, host_dns, continue=false)
+  STDOUT.puts "sshing to #{host_dns}"
+  result = nil
+  tail_command ="tail -f -n1 /var/log/messages"
+  expect = /RightLink.*RS> ([completed|failed]+: < #{recipe} >)/
+
+  Net::SSH.start(host_dns, 'root', :keys => ['~/.ssh/publish-test']) do |ssh|
+    STDOUT.puts ssh.exec!("rs_run_recipe -n '#{recipe}'")
+    log_channel = ssh.open_channel do |ch|
+      ch.exec tail_command do |ch, success|
+        raise "could not execute command" unless success
+        # "on_data" is called when the process writes something to stdout
+        ch.on_data do |c, data|
+          STDOUT.print data
+          if data =~ expect
+            STDOUT.puts "FOUND EXPECTED DATA, closing channel"
+            result = $1
+          end
+        end
+        # "on_extended_data" is called when the process writes something to stderr
+        ch.on_extended_data do |c, type, data|
+          STDERR.print data
+        end
+        ch.on_close do 
+          STDOUT.puts "closed channel" 
+        end
+        ch.on_process do |c|
+          if result
+            STDOUT.puts "attempting close"
+            ch.close
+            ssh.exec("killall tail")
+          end
+        end
+      end
+    end 
+    log_channel.wait
+  end
+  raise "FATAL: halting execution, script #{result}" if result.include?('failed') && continue == false
+  return result
+end
+
+def spot_check(command, host_dns, &block)
+  Net::SSH.start(host_dns, 'root', :keys => ['~/.ssh/publish-test']) do |ssh|
+    result = ssh.exec!(command)
+    yield result
+  end
+end
+
+opts = Trollop::options do
+  opt :deployment, "deployment nickname", :type => :string, :required => false
+  opt :id, "deployment id", :type => :string, :required => false
+  opt :only, "regex string matching the nickname of the servers you want to relaunch. This excludes servers that do not match\nExample --only ubuntu", :type => :string, :required => false
+end
+if opts[:id]
+  deployment = Deployment.find_by_id(opts[:id])
+else
+  deployment = Deployment.find_by_nickname_speed(opts[:deployment]).first
+end
+servers = deployment.servers_no_reload
+servers = servers.select { |s| s.nickname =~ /#{opts[:only]}/ } if opts[:only]
+
+raise "need at least 4 servers to start, only have: #{servers.size}" if servers.size < 4
+# wait for servers to be ready
+servers.each do |s|
+  while(1)
+    puts "waiting for dns-name for #{s.nickname}"
+    break if s['dns-name'] && !s['dns-name'].empty?
+    s.reload
+   sleep 2
+  end
+  puts "DNS: #{s['dns-name']}"
+
+  s.wait_for_state('operational')
+end  
+
+servers.each do |s|
+  puts run_recipe("lb_haproxy::do_attach_request", s.dns_name)
+end
+sleep 20
+
+a_frontend = servers.detect {|d| d.nickname.include?("FE")}
+a_frontend.reload
+raise "couldn't find a frontend with FE in the nickname!" unless a_frontend
+puts a_frontend.dns_name
+spot_check("cat /home/haproxy/rightscale_lb.cfg |grep server|grep -v '#'|wc -l", a_frontend.dns_name) do |result|
+  puts "found attached servers: #{result}"
+  raise "not enough servers #{result}" unless result.to_i == 4
+end
+
+servers.each do |s|
+  run_recipe("app_php::do_update_code", s.dns_name)
+end
+
+servers.each do |s|
+  run_recipe("lb_haproxy::do_detach_request", s.dns_name)
+end
+sleep 20
+spot_check("cat /home/haproxy/rightscale_lb.cfg |grep server|grep -v '#'|wc -l", a_frontend.dns_name) do |result|
+  puts "found attached servers: #{result}"
+  raise "should be zero servers but was #{result}" unless result.to_i == 0
+end
+
+